From 56ec194862ff5f9f764503b84f73aa22e9e0bf61 Mon Sep 17 00:00:00 2001 From: Grzegorz Kowal Date: Fri, 9 Feb 2018 15:25:39 -0200 Subject: [PATCH 1/5] IO: Store block IDs in coordinate group. Block IDs might be useful for block diagnostic purposes. Signed-off-by: Grzegorz Kowal --- src/io.F90 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/io.F90 b/src/io.F90 index f06669f..add981f 100644 --- a/src/io.F90 +++ b/src/io.F90 @@ -2653,7 +2653,7 @@ module io ! local allocatable arrays ! - integer(kind=4), dimension(:) , allocatable :: lev, ref + integer(kind=4), dimension(:) , allocatable :: ids, lev, ref integer(kind=4), dimension(:,:) , allocatable :: cor real (kind=8), dimension(:,:,:), allocatable :: bnd @@ -2688,6 +2688,7 @@ module io ! allocate arrays to store coordinates ! + allocate(ids(cm(1))) allocate(lev(cm(1))) allocate(ref(cm(1))) allocate(cor(cm(1),cm(2))) @@ -2699,6 +2700,10 @@ module io pdata => list_data do while(associated(pdata)) +! fill in the IDs array +! + ids(l) = pdata%meta%id + ! fill in the level array ! lev(l) = pdata%meta%level @@ -2728,6 +2733,7 @@ module io ! write the arrays to the HDF5 file ! + call write_array(gid, 'ids' , cm(1), ids) call write_array(gid, 'levels', cm(1), lev) call write_array(gid, 'refine', cm(1), ref) call write_array(gid, 'coords', cm(:), cor) @@ -2738,6 +2744,7 @@ module io ! deallocate temporary arrays ! + if (allocated(ids)) deallocate(ids) if (allocated(lev)) deallocate(lev) if (allocated(ref)) deallocate(ref) if (allocated(cor)) deallocate(cor) From eef631de500cf9631043c116b80b876d7108210b Mon Sep 17 00:00:00 2001 From: Grzegorz Kowal Date: Fri, 9 Feb 2018 16:44:19 -0200 Subject: [PATCH 2/5] RANDOM: Rewrite seed numbers handling. The array of seed numbers makes sense only when OpenMP is used. With MPI on, the length of seed vector on each MPI process is equal to the number of OpenMP threads. For no OpenMP, only one seed number per MPI process is used. Signed-off-by: Grzegorz Kowal --- src/driver.F90 | 5 ++-- src/random.F90 | 70 ++++++++++++++++++-------------------------------- 2 files changed, 28 insertions(+), 47 deletions(-) diff --git a/src/driver.F90 b/src/driver.F90 index cbf20b5..ee37f85 100644 --- a/src/driver.F90 +++ b/src/driver.F90 @@ -308,9 +308,10 @@ program amun ! call setup_mpi(div(:), per(:), .false.) -! initialize the random number generator +! initialize the random number generator (passes the number of OpenMP threads +! and the current thread number) ! - call initialize_random(nprocs, nproc) + call initialize_random(1, 0) ! initialize geometry modules and print info ! diff --git a/src/random.F90 b/src/random.F90 index 451dbb2..14790f1 100644 --- a/src/random.F90 +++ b/src/random.F90 @@ -80,7 +80,7 @@ module random ! !=============================================================================== ! - subroutine initialize_random(nprocs, nproc) + subroutine initialize_random(nthreads, nthread) ! obtain required variables from other modules ! @@ -92,7 +92,7 @@ module random ! subroutine arguments ! - integer, intent(in) :: nprocs, nproc + integer, intent(in) :: nthreads, nthread ! local variables ! @@ -112,9 +112,9 @@ module random call start_timer(iri) #endif /* PROFILE */ -! set the processor number +! set the thread number ! - kp = nproc + kp = nthread ! obtain the generator type ! @@ -122,7 +122,7 @@ module random ! calculate the number of seeds ! - nseeds = 2 * nprocs + nseeds = nthreads lseed = nseeds - 1 ! allocate seeds for random number generator @@ -138,12 +138,8 @@ module random seeds(i) = 123456789 * r end do case default - call random_number(r) - do i = 0, nprocs - 1 - seeds(i) = 123456789 * r - end do - call random_number(r) - do i = nprocs, lseed + r = 0.1234567890123456789 + do i = 0, lseed seeds(i) = 123456789 * r end do end select @@ -204,7 +200,7 @@ module random ! !=============================================================================== ! - subroutine set_seeds(np, seed) + subroutine set_seeds(np, nseeds) ! declare all variables as implicit ! @@ -212,8 +208,8 @@ module random ! input arguments ! - integer , intent(in) :: np - integer(kind=4), dimension(0:np-1), intent(in) :: seed + integer , intent(in) :: np + integer(kind=4), dimension(np), intent(in) :: nseeds ! local variables ! @@ -230,35 +226,19 @@ module random ! set the seeds only if the input array and seeds have the same sizes ! - if (np == nseeds) then - - seeds(0:lseed) = seed(0:lseed) - - else - -! if the input array and seeds have different sizes, expand or shrink seeds -! - select case(gentype) - case('random') - l = min(lseed, np - 1) - seeds(0:l) = seed(0:l) - if (l < lseed) then - do i = l + 1, lseed - call random_number(r) - seeds(i) = 123456789 * r - end do - end if - case default - l = nseeds / 2 - do i = 0, l - 1 - seeds(i) = seed(0) + l = min(lseed, np - 1) + seeds(0:l) = seed(1:l+1) + select case(gentype) + case('random') + if (l < lseed) then + do i = l + 1, lseed + call random_number(r) + seeds(i) = 123456789 * r end do - do i = l, lseed - seeds(i) = seed(np-1) - end do - end select - - end if + end if + case default + seeds(l+1:lseed) = seeds(0) + end select #ifdef PROFILE ! stop accounting time for the random number generator @@ -279,7 +259,7 @@ module random ! !=============================================================================== ! - subroutine get_seeds(seed) + subroutine get_seeds(rseeds) ! declare all variables as implicit ! @@ -287,7 +267,7 @@ module random ! output arguments ! - integer(kind=4), dimension(0:lseed), intent(out) :: seed + integer(kind=4), dimension(nseeds), intent(out) :: rseeds ! !------------------------------------------------------------------------------- ! @@ -297,7 +277,7 @@ module random call start_timer(iri) #endif /* PROFILE */ - seed(0:lseed) = seeds(0:lseed) + rseeds(1:nseeds) = seeds(0:lseed) #ifdef PROFILE ! stop accounting time for the random number generator From cdd9b8e204f53d4df3798656ea223a475b60478b Mon Sep 17 00:00:00 2001 From: Grzegorz Kowal Date: Fri, 9 Feb 2018 16:53:33 -0200 Subject: [PATCH 3/5] RANDOM: Correct compilation error in set_seeds(). Signed-off-by: Grzegorz Kowal --- src/random.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/random.F90 b/src/random.F90 index 14790f1..6e6b70a 100644 --- a/src/random.F90 +++ b/src/random.F90 @@ -227,7 +227,7 @@ module random ! set the seeds only if the input array and seeds have the same sizes ! l = min(lseed, np - 1) - seeds(0:l) = seed(1:l+1) + seeds(0:l) = nseeds(1:l+1) select case(gentype) case('random') if (l < lseed) then From 14b7cbd9ffaf2ca3c6edf88d38b7e305fbff6ecb Mon Sep 17 00:00:00 2001 From: Grzegorz Kowal Date: Fri, 9 Feb 2018 17:25:04 -0200 Subject: [PATCH 4/5] IO: Restore metadata from files corresponding to process number. Signed-off-by: Grzegorz Kowal --- src/io.F90 | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/io.F90 b/src/io.F90 index add981f..66360e7 100644 --- a/src/io.F90 +++ b/src/io.F90 @@ -796,13 +796,17 @@ module io !! 1. RESTORE PARAMETERS AND META BLOCKS FROM THE FIRST FILE !! -! prepare the filename +! prepare the filename using the current process number; in case the file does +! not exist decrease it until the file corresponding to lower process number +! is found; ! - write (fl, "(a,'r',i6.6,'_',i5.5,'.h5')") trim(respath), nrest, 0 - -! check if the HDF5 file exists -! - inquire(file = fl, exist = info) + info = .false. + lfile = nproc + 1 + do while (.not. info .and. lfile > 0) + lfile = lfile - 1 + write (fl, "(a,'r',i6.6,'_',i5.5,'.h5')") trim(respath), nrest, lfile + inquire(file = fl, exist = info) + end do ! quit, if file does not exist ! From 371b035ba87ba29d29964b147136138b0847da8f Mon Sep 17 00:00:00 2001 From: Grzegorz Kowal Date: Fri, 9 Feb 2018 17:37:19 -0200 Subject: [PATCH 5/5] IO, RANDOM: Generate new seeds for processes > restart files. If we increase the number of MPI processes during the restart, the seed values are read from the last available restart file. Therefore, if we have completely random seed numbers, the extra processes will have the same seed value. Detect this case and generate new random seeds if necessary. Signed-off-by: Grzegorz Kowal --- src/io.F90 | 2 +- src/random.F90 | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/io.F90 b/src/io.F90 index 66360e7..7b90c4b 100644 --- a/src/io.F90 +++ b/src/io.F90 @@ -1571,7 +1571,7 @@ module io ! set the seed values ! - call set_seeds(lnseeds, seeds(:)) + call set_seeds(lnseeds, seeds(:), nproc /= lnproc) ! deallocate seed array ! diff --git a/src/random.F90 b/src/random.F90 index 6e6b70a..62ecd51 100644 --- a/src/random.F90 +++ b/src/random.F90 @@ -200,7 +200,7 @@ module random ! !=============================================================================== ! - subroutine set_seeds(np, nseeds) + subroutine set_seeds(np, nseeds, generate) ! declare all variables as implicit ! @@ -210,6 +210,7 @@ module random ! integer , intent(in) :: np integer(kind=4), dimension(np), intent(in) :: nseeds + logical , intent(in) :: generate ! local variables ! @@ -230,7 +231,12 @@ module random seeds(0:l) = nseeds(1:l+1) select case(gentype) case('random') - if (l < lseed) then + if (generate) then + do i = 0, lseed + call random_number(r) + seeds(i) = 123456789 * r + end do + else do i = l + 1, lseed call random_number(r) seeds(i) = 123456789 * r