# Parameters for the Musubi recheck runs.

import os
import sys
import datetime
import shutil

from clone_and_build_function import *

templateFolder = './templates/'
machineFolder = './machines/'
apesFolder = os.getenv('HOME')+'/apes/'

date = datetime.datetime.now().strftime("%Y-%m-%d__%X")
weekday = datetime.datetime.now().strftime("%A")

# Production directory, keep the past week as history.
prod_dir = 'musubi-runs_' + weekday

run_label = 'MUSUBI'

# Cleanup production directory before using it:
shutil.rmtree(prod_dir, ignore_errors=True)
loglevel = 'INFO'

hg_clone_source = 'ssh://geb.sts.nt.uni-siegen.de//var/www/repos/hg/'

from recheck import notify_list
mail_address = notify_list

musubi_path = apesFolder + 'musubi/testsuite/'

# name of the shepherd log file
shepherd_out = 'shepherd.log'

# name of the log and rror file of the clone and build function
clone_build_out = 'clone_build.log'
clone_build_err = 'clone_build_error.log'

create_tag_on = True
grep_performance = True 
loris_clone_url = apesFolder + 'loris/'

shepherd_jobs = []

# Use the latest revision
seeder_exe = clone_build( solver          = 'seeder',
        revision        = 'default',
        hg_clone_source = hg_clone_source+'seeder',
        solver_dir      = 'seeder',
        clone_build_out = clone_build_out,
        clone_build_err = clone_build_err         )

musubi_exe = clone_build( solver          = 'musubi',
        hg_clone_source = hg_clone_source+'musubi',
        solver_dir      = 'musubi',
        # confopts        = '--openmp',
        clone_build_out = clone_build_out,
        clone_build_err = clone_build_err         )

musubi_omp = clone_build( solver          = 'musubi',
        hg_clone_source = hg_clone_source+'musubi',
        solver_dir      = 'musubi',
        confopts        = '--openmp',
        clone_build_out = clone_build_out,
        clone_build_err = clone_build_err         )

### MUSUBI JOB 1 - Lid Cavity
testcase_path = musubi_path+'validation/Lid-driven_Cavity/'
shepherd_jobs.append(dict(executable = seeder_exe,
    template=testcase_path+'seeder.lua',
    extension='lua',
    run_exec = True,
    create_subdir = ['mesh'],
    prefix = 'lidcavity',
    label = 'lidcavity_seeder',
    ))
shepherd_jobs.append(dict(executable = musubi_exe,
    solver_name = 'musubi',
    template=testcase_path+'musubi.lua',
    extension='lua',
    run_exec = True,
    additional_params = dict(testcase_path = testcase_path),
    create_subdir = ['tracking'],
    depend = ['lidcavity_seeder'],
                          create_dir = False,
                          label = 'lidcavity_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_lidcavity_horizondalLine_vel_p00000_t135.450E+03.res',
                          val_output_filename = 'lidcavity_horizondalLine_vel_p00000_t135.591E+03.res',
                          ))


## MUSUBI JOB 2  - Flow Around Cylinder 2D bench
testcase_path = musubi_path+'validation/FlowAroundCylinder2D_Bench/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'FlowAroundCylinder2D',
                          label = 'FlowAroundCylinder2D_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          create_dir=False,
                          create_subdir = ['mesh'],
                          depend = ['FlowAroundCylinder2D_common'],
                          label = 'FlowAroundCylinder2D_seeder',
                          attachment = True,
                         ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -np 8',
                          create_subdir = ['tracking'],
                          depend = ['FlowAroundCylinder2D_seeder'],
                          create_dir=False,
                          label = 'FlowAroundCylinder2D_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_FlowAroundCyl_probe_pressure_p00000.res',
                          val_output_filename = 'FlowAroundCyl_probe_pressure_p00000.res',
                          ))

### MUSUBI JOB 3  - Mini Channel
testcase_path = musubi_path+'physics/mini_channel/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path+'stl/'),
                          create_subdir = ['mesh'],
                          prefix = 'mini_channel',
                          label = 'mini_channel_seeder',
                          attachment = True,
                          mail=True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking'],
                          depend = ['mini_channel_seeder'],
                          create_dir=False,
                          label = 'mini_channel_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_mini-channel_probePress_p00000.res',
                          val_output_filename = 'mini-channel_probePress_p00000.res',
                          ))

#### MUSUBI JOB 4  - simple channel
testcase_path = musubi_path+'physics/simple_channel/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['mesh'],
                          prefix = 'simple_channel_vali',
                          label = 'simple_channel_vali_seeder',
                          attachment = True,
                          mail = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking'],
                          depend = ['simple_channel_vali_seeder'],
                          create_dir=False,
                          label = 'simple_channel_vali_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'tracking/ref_simple_channel_probe_density_p00000.res',
                          val_output_filename = 'tracking/simple_channel_probe_density_p00000.res',
                          ))

### MUSUBI JOB  5 - spacer 2D
testcase_path = musubi_path+'physics/spacer2D/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['mesh'],
                          prefix = 'spacer2D',
                          label = 'spacer2D_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          depend = ['spacer2D_seeder'],
                          create_dir=False,
                          label = 'spacer2D_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_spacer2D_probe_vel_p00000.res',
                          val_output_filename = 'spacer2D_probe_vel_p00000.res',
                          ))

### MUSUBI JOB 6  - channel cylinder multilevel
# testcase_path = musubi_path+'physics/channel_cylinder_multilevel/'
# shepherd_jobs.append(dict(executable = seeder_exe,
#                           template=testcase_path+'seeder.lua',
#                           extension='lua',
#                           run_exec = True,
#                           additional_params = dict(stl_path = testcase_path+'stl/'),
#                           create_subdir = ['mesh'],
#                           prefix = 'channel_cylinder_acst_qua',
#                           label = 'channel_cylinder_acst_qua_seeder',
#                           attachment = True,
#                           ))
# shepherd_jobs.append(dict(executable = musubi_exe,
#                           solver_name = 'musubi',
#                           template=testcase_path+'musubi.lua',
#                           extension='lua',
#                           run_exec = True,
#                           create_subdir = ['tracking'],
#                           depend = ['channel_cylinder_acst_qua_seeder'],
#                           create_dir=False,
#                           create_depend_path = True,
#                           label = 'channel_cylinder_acst_qua_musubi',
#                           attachment = True,
#                           validation = True,
#                           val_method = 'difference',
#                           val_ref_path = testcase_path+'tracking/ref_channelRefine_lbmmrtacoustic_reference_p00000.res',
#                           val_output_filename = 'tracking/channelRefine_lbmmrtacoustic_reference_p00000.res',
#                           ))

### MUSUBI JOB 7  - TGV 2D
testcase_path = musubi_path+'validation/TGV_2D/'
shepherd_jobs.append(dict(
    executable = musubi_exe,
    solver_name = 'musubi',
    template=testcase_path+'musubi.lua',
    extension='lua',
    run_exec = True,
    additional_params = dict(testcase_path = testcase_path),
    prefix = 'TGV_2D_MRT',
    label = 'TGV_2D_MRT',
    attachment = True,
    validation = True,
    val_method = 'difference',
    position = [ 2,4,5,10 ],
    val_ref_path = testcase_path+'ref_L6_global_l2norm_p00000.res',
    val_output_filename = 'L6_global_l2norm_p00000.res',
    ))


### MUSUBI JOB 8  - nrbc
testcase_path = musubi_path+'physics/nrbc/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['mesh'],
                          prefix = 'nrbc',
                          label = 'nrbc_seeder',
                          attachment = True,
                         ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking','restart'],
                          create_dir=False,
                          depend = ['nrbc_seeder'],
                          create_depend_path = True,
                          label = 'nrbc_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'tracking/ref_gaussian_probe_lbm_p00000.res',
                          val_output_filename = 'tracking/gaussian_probe_fluid_p00000.res',
                          ))

### MUSUBI JOB 9  - nonNewtonian
testcase_path = musubi_path+'verification/nonNewtonian/channel2D/power_law/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'nonNewtonian-power-law',
                          label = 'nonNewtonian-power-law_common',
                         ))

shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['nonNewtonian-power-law_common'],
                          create_dir=False,
                          label = 'nonNewtonian-power-law_seeder',
                          attachment = True,
                         ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['tracking'],
                          depend = ['nonNewtonian-power-law_seeder'],
                          create_dir=False,
                          label = 'nonNewtonian-power-law_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'tracking/ref_nonNewtonian_Power_Law_vel_l2norm_p00000.res',
                          val_output_filename = 'tracking/nonNewtonian_vel_l2norm_p00000.res',
                          ))

### MUSUBI JOB 10  - Channel inclined
testcase_path = musubi_path+'validation/channel_inclined/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path+'stl/'),
                          create_subdir = ['mesh'],
                          prefix = 'wall_inclined',
                          label = 'wall_inclined_seeder',
                          attachment = True,
                         ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking'],
                          depend = ['wall_inclined_seeder'],
                          create_dir=False,
                          create_depend_path = True,
                          label = 'wall_inclined_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'tracking/ref_channelIncli_velErr8_p00000.res',
                          val_output_filename = 'tracking/channelIncli_velErr8_p00000.res',
                          ))


### MUSUBI JOB 11 
testcase_path = musubi_path+'validation/multispecies_gasMix/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'Multispecies_gasMix',
                          label = 'Multispecies_gasMix_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['Multispecies_gasMix_common'],
                          create_dir=False,
                          label = 'Multispecies_gasMix_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['Multispecies_gasMix_seeder'],
                          create_dir=False,
                          label = 'Multispecies_gasMix_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_multispecies_line_p00000_t1.000E+03.res',
                          val_output_filename = 'multispecies_line_p00000_t1.000E+03.res',
                          ))


### MUSUBI JOB 12 
testcase_path = musubi_path+'validation/liquidModel/solventLiquidMix/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'MSLM_Solvent',
                          label = 'MSLM_Solvent_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['MSLM_Solvent_common'],
                          create_dir=False,
                          label = 'MSLM_Solvent_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['MSLM_Solvent_seeder'],
                          create_dir=False,
                          label = 'MSLM_Solvent_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_multispecies_point_p00000.res',
                          val_output_filename = 'multispecies_point_p00000.res',
                          ))

### MUSUBI JOB 13 
testcase_path = musubi_path+'validation/liquidModel/dilute_externalForce/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'MSLM_diluteWithExtForce',
                          label = 'MSLM_diluteWithExtForce_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['MSLM_diluteWithExtForce_common'],
                          create_dir=False,
                          label = 'MSLM_diluteWithExtForce_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['MSLM_diluteWithExtForce_seeder'],
                          create_dir=False,
                          label = 'MSLM_diluteWithExtForce_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_diluteWithForce_point_p00000.res',
                          val_output_filename = 'diluteWithForce_point_p00000.res',
                          ))

### MUSUBI JOB 14 
testcase_path = musubi_path+'validation/liquidModel/taylorGreenVortex/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'MSLM_TGV',
                          label = 'MSLM_TGV_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['MSLM_TGV_common'],
                          create_dir=False,
                          label = 'MSLM_TGV_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['MSLM_TGV_seeder'],
                          create_dir=False,
                          label = 'MSLM_TGV_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_TGV_MS_pointVel_p00000.res',
                          val_output_filename = 'TGV_MS_pointVel_p00000.res',
                          ))

### MUSUBI job 16: BGK performance using cube
testcase_path = musubi_path+'performance/parallel/TGV_3D_single_level/'
shepherd_jobs.append(dict(
    executable = musubi_exe,
    solver_name = 'musubi',
    template=testcase_path+'bgk_d3q19.lua',
    extension='lua',
    run_exec = True,
    run_env = dict(OMP_NUM_THREADS='3'),
    run_command = 'mpirun -n 1',
    additional_params = dict(testcase_path = testcase_path),
    create_subdir = ['tracking'],
    prefix = 'bgk_d3q19',
    label = 'bgk_d3q19',
    attachment = True,
    validation = True,
    val_method = 'difference',
    position = [2],
    val_ref_path = testcase_path+'tracking/ref_l7_bgk_d3q19_total_density_p00000.res',
    val_output_filename =        'tracking/l7_bgk_d3q19_total_density_p00000.res',
    ))

### MUSUBI job 17: TRT performance using cube
testcase_path = musubi_path+'performance/parallel/TGV_3D_single_level/'
shepherd_jobs.append(dict(
    executable = musubi_exe,
    solver_name = 'musubi',
    template=testcase_path+'trt_d3q19.lua',
    extension='lua',
    run_exec = True,
    run_command = 'mpirun -n 3',
    additional_params = dict(testcase_path = testcase_path),
    create_subdir = ['tracking'],
    prefix = 'trt_d3q19',
    label = 'trt_d3q19',
    attachment = True,
    validation = True,
    val_method = 'difference',
    position = [2],
    val_ref_path = testcase_path+'tracking/ref_l7_trt_d3q19_total_density_p00000.res',
    val_output_filename =        'tracking/l7_trt_d3q19_total_density_p00000.res',
    ))

### MUSUBI job 18: MRT performance using cube
testcase_path = musubi_path+'performance/parallel/TGV_3D_single_level/'
shepherd_jobs.append(dict(
    executable = musubi_exe,
    solver_name = 'musubi',
    template=testcase_path+'mrt_d3q19.lua',
    extension='lua',
    run_exec = True,
    run_command = 'mpirun -n 3',
    additional_params = dict(testcase_path = testcase_path),
    create_subdir = ['tracking'],
    prefix = 'mrt_d3q19',
    label = 'mrt_d3q19',
    attachment = True,
    validation = True,
    val_method = 'difference',
    position = [2],
    val_ref_path = testcase_path+'tracking/ref_l7_mrt_d3q19_total_density_p00000.res',
    val_output_filename =        'tracking/l7_mrt_d3q19_total_density_p00000.res',
    ))

### MUSUBI job 19: BGK GC D3Q27 performance using cube
testcase_path = musubi_path+'performance/parallel/TGV_3D_single_level/'
shepherd_jobs.append(dict(
    executable = musubi_omp,
    solver_name = 'musubi',
    template=testcase_path+'bgk_d3q27.lua',
    extension='lua',
    run_exec = True,
    run_env = dict(OMP_NUM_THREADS='3'),
    run_command = 'mpirun -n 1',
    additional_params = dict(testcase_path = testcase_path),
    create_subdir = ['tracking'],
    prefix = 'bgk_d3q27',
    label = 'bgk_d3q27',
    attachment = True,
    validation = True,
    val_method = 'difference',
    position = [2],
    val_ref_path = testcase_path+'tracking/ref_l7_bgk_d3q27_total_density_p00000.res',
    val_output_filename =        'tracking/l7_bgk_d3q27_total_density_p00000.res',
    ))

### MUSUBI JOB 20: channel cylinder Re20 force calculation
testcase_path = musubi_path+'validation/channel_cylinder/Re20/'
shepherd_jobs.append(dict(
    executable = None,
    mail = False,
    template=testcase_path+'common.lua',
    extension='lua',
    run_exec = False,
    prefix = 'cylinder_re20',
    label = 'cylinder_re20_common',
    ))
shepherd_jobs.append(dict(
    executable = seeder_exe,
    template=testcase_path+'seeder.lua',
    additional_params = dict(stl_path = testcase_path+'../'),
    extension='lua',
    run_exec = True,
    create_dir=False,
    depend = ['cylinder_re20_common'],
    create_subdir = ['mesh_L10'],
    label = 'cylinder_re20_seeder',
    ))
shepherd_jobs.append(dict(
    executable = musubi_exe,
    solver_name = 'musubi',
    template=testcase_path+'musubi.lua',
    extension='lua',
    run_exec = True,
    run_command = 'mpirun -npersocket 8',
    additional_params = dict(testcase_path = testcase_path),
    create_subdir = ['tracking'],
    depend = ['cylinder_re20_seeder'],
    create_dir = False,
    label = 'cylinder_re20_musubi',
    attachment = True,
    validation = True,
    val_method = 'difference',
    position = [2,3],
    val_ref_path = testcase_path+'tracking/ref_L10_force_on_cylinder_p00000.res',
    val_output_filename =        'tracking/L10_force_on_cylinder_p00000.res',
    ))

### MUSUBI JOB 21: channel cylinder Re20 force calculation with nonEquilbrium Bnd
testcase_path = musubi_path+'validation/channel_cylinder/Re20_nonEqBC/'
shepherd_jobs.append(dict(
    executable = None,
    mail = False,
    template=testcase_path+'common.lua',
    extension='lua',
    run_exec = False,
    prefix = 'cylinder_re20_nonEqBC',
    label = 'cylinder_re20_nonEqBC_common',
    ))
shepherd_jobs.append(dict(
    executable = seeder_exe,
    template=testcase_path+'seeder.lua',
    additional_params = dict(stl_path = testcase_path+'../'),
    extension='lua',
    run_exec = True,
    create_dir=False,
    depend = ['cylinder_re20_nonEqBC_common'],
    create_subdir = ['mesh_L10'],
    label = 'cylinder_re20_nonEqBC_seeder',
    ))
shepherd_jobs.append(dict(
    executable = musubi_exe,
    solver_name = 'musubi',
    template=testcase_path+'musubi.lua',
    extension='lua',
    run_exec = True,
    run_command = 'mpirun -npersocket 8',
    additional_params = dict(testcase_path = testcase_path),
    create_subdir = ['tracking'],
    depend = ['cylinder_re20_nonEqBC_seeder'],
    create_dir = False,
    label = 'cylinder_re20_nonEqBC_musubi',
    attachment = True,
    validation = True,
    val_method = 'difference',
    position = [2,3],
    val_ref_path = testcase_path+'tracking/ref_L10_force_on_cylinder_p00000.res',
    val_output_filename =        'tracking/L10_force_on_cylinder_p00000.res',
    ))

### MUSUBI JOB 22: Multispecies liquid model taylor dispersion with molefraction inlet
### and velocity outlet BC
testcase_path = musubi_path+'validation/liquidModel/taylorDispersion/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'MSLM_taylorDisp',
                          label = 'MSLM_taylorDisp_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['MSLM_taylorDisp_common'],
                          create_dir=False,
                          label = 'MSLM_taylorDisp_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['MSLM_taylorDisp_seeder'],
                          create_dir=False,
                          label = 'MSLM_taylorDisp_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_taylorDispersion_mole_density_p00000_t2.000E+00.res',
                          val_output_filename = 'taylorDispersion_mole_density_p00000_t2.000E+00.res',
                          ))

### MUSUBI JOB 23: Multispecies liquid model taylor dispersion with external force
testcase_path = musubi_path+'validation/liquidModel/taylorDispersion_externalForce/'
shepherd_jobs.append(dict(executable = None, 
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'MSLM_taylorDisp_ExtForce',
                          label = 'MSLM_taylorDisp_ExtForce_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['MSLM_taylorDisp_ExtForce_common'],
                          create_dir=False,
                          label = 'MSLM_taylorDisp_ExtForce_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -np 8',
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['MSLM_taylorDisp_ExtForce_seeder'],
                          create_dir=False,
                          label = 'MSLM_taylorDisp_ExtForce_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_taylorDispersion_moleDensity_p00000.res',
                          val_output_filename = 'taylorDispersion_moleDensity_p00000.res',
                          ))

### MUSUBI JOB 24: Multispecies liquid model stefan tube
testcase_path = musubi_path+'validation/liquidModel/stefanTube/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'MSLM_stefanTube',
                          label = 'MSLM_stefanTube_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['MSLM_stefanTube_common'],
                          create_dir=False,
                          label = 'MSLM_stefanTube_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['MSLM_stefanTube_seeder'],
                          create_dir=False,
                          label = 'MSLM_stefanTube_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_stefantube_line_p00000_t1.000E+03.res',
                          val_output_filename = 'stefantube_line_p00000_t1.000E+03.res',
                          ))

### MUSUBI JOB 25: Multispecies liquid model with spacer
testcase_path = musubi_path+'validation/liquidModel/spacerLab_externalForce/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'MSLM_spacer',
                          label = 'MSLM_spacer_common',
                         ))


shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['MSLM_spacer_common'],
                          create_dir=False,
                          label = 'MSLM_spacer_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -n 8',
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['MSLM_spacer_seeder'],
                          create_dir=False,
                          label = 'MSLM_spacer_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_spacer_moleDensity_p00000.res',
                          val_output_filename = 'spacer_moleDensity_p00000.res',
                          ))

### MUSUBI JOB 26: Womersley flow driven by external force
testcase_path = musubi_path+'validation/womersley/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'complex.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'womersley',
                          label = 'womersley_complex',
                         ))
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'womersley',
                          label = 'womersley_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['womersley_common'],
                          create_dir=False,
                          label = 'womersley_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['womersley_seeder'],
                          create_dir=False,
                          create_subdir = ['tracking'],
                          label = 'womersley_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
 val_ref_path = testcase_path+'tracking/ref_womersley2D_total_density_p00000.res',
 val_output_filename = 'tracking/womersley2D_total_density_p00000.res',
                          ))

### MUSUBI JOB 27: Cube two levels mesh with balance
testcase_path = musubi_path+'balance/cube/two_levels/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'cube_two_levels_balance',
                          label = 'cube_two_levels_balance_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh_L5', 'balance'],
                          depend = ['cube_two_levels_balance_common'],
                          create_dir=False,
                          label = 'cube_two_levels_balance_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -n 3',
                          additional_params = dict(testcase_path = testcase_path),
                          depend = ['cube_two_levels_balance_seeder'],
                          create_dir=False,
                          label = 'cube two levels balance',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_point_density.res',
                          val_output_filename = 'cube_two_levels_L5_point_density_p00000.res',
                          ))

### MUSUBI JOB 28: Free flow nozzle test case used in simTech II 
testcase_path = musubi_path+'physics/free_flow_nozzle/simTechII/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'free_flow_nozzle',
                          label = 'free_flow_nozzle_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['mesh'],
                          depend = ['free_flow_nozzle_common'],
                          create_dir=False,
                          label = 'free_flow_nozzle_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -n 8',
                          additional_params = dict(testcase_path = testcase_path),
                          create_subdir = ['tracking'],
                          depend = ['free_flow_nozzle_seeder'],
                          create_dir=False,
                          label = 'free_flow_nozzle_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'tracking/ref_nozzle_probe_neck_p00000.res',
                          val_output_filename = 'tracking/nozzle_probe_neck_p00000.res',
                          ))

### MUSUBI JOB 29: sphere single level mesh with balance
testcase_path = musubi_path+'balance/sphere/single_level/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'sphere_single_level_balance',
                          label = 'sphere_single_level_balance_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path),
                          create_subdir = ['mesh_L5'],
                          depend = ['sphere_single_level_balance_common'],
                          create_dir=False,
                          label = 'sphere_single_level_balance_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -n 3',
                          # additional_params = dict(testcase_path = testcase_path),
                          depend = ['sphere_single_level_balance_seeder'],
                          create_dir=False,
                          label = 'sphere single level balance',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_L5_global_density_p00000.res',
                          val_output_filename = 'L5_global_density_p00000.res',
                          ))

### MUSUBI JOB 30: mutlilevel channel2D: quadratic interpolation, incompressible, D2Q9
testcase_path = musubi_path+'validation/channel2D_refinePatches/2D/quad_incomp/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path),
                          create_subdir = ['mesh'],
                          prefix = 'channel_2D_quad_incomp',
                          label = 'channel_2D_quad_incomp_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking'],
                          run_command = 'mpirun -n 1',
                          depend = ['channel_2D_quad_incomp_seeder'],
                          create_dir=False,
                          label = 'channel_2D_quad_incomp',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref/channel2D_quad_incomp_ref_line_p00000_t20.187E+00.res',
                          val_output_filename = 'tracking/channel2D_quad_incomp_ref_line_p00000_t20.187E+00.res',
                          ))

### MUSUBI JOB 31: mutlilevel channel2D: weighted average interpolation, compressible, D2Q9
testcase_path = musubi_path+'validation/channel2D_refinePatches/2D/avg_comp/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path),
                          create_subdir = ['mesh'],
                          prefix = 'channel_2D_avg_comp',
                          label = 'channel_2D_avg_comp_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking'],
                          run_command = 'mpirun -n 1',
                          depend = ['channel_2D_avg_comp_seeder'],
                          create_dir=False,
                          label = 'channel_2D_avg_comp',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref/channel2D_avg_comp_ref_line_p00000_t37.095E+00.res',
                          val_output_filename = 'tracking/channel2D_avg_comp_ref_line_p00000_t37.095E+00.res',
                          ))

### MUSUBI JOB 32: mutlilevel channel2D: weighted average interpolation, incompressible, D3Q19
testcase_path = musubi_path+'validation/channel2D_refinePatches/3D/D3Q19/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path),
                          create_subdir = ['mesh'],
                          prefix = 'channel_2D_D3Q19',
                          label = 'channel_2D_D3Q19_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking'],
                          run_command = 'mpirun -n 1',
                          depend = ['channel_2D_D3Q19_seeder'],
                          create_dir=False,
                          label = 'channel_2D_D3Q19',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref/channel_D3Q19_avg_incomp_ref_line_p00000_t19.886E+00.res',
                          val_output_filename = 'tracking/channel_D3Q19_avg_incomp_ref_line_p00000_t19.886E+00.res',
                          ))

### MUSUBI JOB 33: mutlilevel channel2D: weighted average interpolation, incompressible, D3Q19
testcase_path = musubi_path+'validation/channel2D_refinePatches/3D/D3Q27/'
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path),
                          create_subdir = ['mesh'],
                          prefix = 'channel_2D_D3Q27',
                          label = 'channel_2D_D3Q27_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['tracking'],
                          run_command = 'mpirun -n 1',
                          depend = ['channel_2D_D3Q27_seeder'],
                          create_dir=False,
                          label = 'channel_2D_D3Q27',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref/channel2D_quad_comp_ref_line_p00000_t39.001E+00.res',
                          val_output_filename = 'tracking/channel2D_quad_comp_ref_line_p00000_t39.001E+00.res',
                          ))

### MUSUBI JOB 34: Electric double layer validation of lbm_pot solver with potential BC
testcase_path = musubi_path+'validation/poisson/electricalDoubleLayer/bc_pot_dirichlet/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'edl_bc_pot',
                          label = 'edl_bc_pot_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['mesh'],
                          depend = ['edl_bc_pot_common'],
                          create_dir=False,
                          label = 'edl_bc_pot_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -n 1',
                          depend = ['edl_bc_pot_seeder'],
                          create_dir=False,
                          label = 'edl_bc_pot_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_EDL_PotBC_line_p00000_t2.247E-15.res',
                          val_output_filename = 'EDL_PotBC_line_p00000_t2.247E-15.res',
                          ))

### MUSUBI JOB 35: Electric double layer validation of lbm_pot solver with neumann BC
testcase_path = musubi_path+'validation/poisson/electricalDoubleLayer/bc_chargeDens_neumann/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'edl_bc_chargedens',
                          label = 'edl_bc_chargedens_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          create_subdir = ['mesh'],
                          depend = ['edl_bc_chargedens_common'],
                          create_dir=False,
                          label = 'edl_bc_chargedens_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -n 1',
                          depend = ['edl_bc_chargedens_seeder'],
                          create_dir=False,
                          label = 'edl_bc_chargedens_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'ref_EDL_chargeDensBC_line_p00000_t2.296E-12.res',
                          val_output_filename = 'EDL_chargeDensBC_line_p00000_t2.296E-12.res',
                          ))
### MUSUBI JOB 36: Concentric cylinders  validation of lbm_pot solver 
testcase_path = musubi_path+'validation/poisson/concentric_cylinders/'
shepherd_jobs.append(dict(executable = None,
                          mail = False,
                          template=testcase_path+'common.lua',
                          extension='lua',
                          run_exec = False,
                          prefix = 'concentric_cyl',
                          label = 'concentric_cyl_common',
                         ))
shepherd_jobs.append(dict(executable = seeder_exe,
                          template=testcase_path+'seeder.lua',
                          extension='lua',
                          run_exec = True,
                          additional_params = dict(stl_path = testcase_path+'/'),
                          create_subdir = ['mesh'],
                          depend = ['concentric_cyl_common'],
                          create_dir=False,
                          label = 'concentric_cyl_seeder',
                          attachment = True,
                          ))
shepherd_jobs.append(dict(executable = musubi_exe,
                          solver_name = 'musubi',
                          template=testcase_path+'musubi.lua',
                          extension='lua',
                          run_exec = True,
                          run_command = 'mpirun -n 8',
                          depend = ['concentric_cyl_seeder'],
                          create_subdir = ['tracking'],
                          create_dir=False,
                          label = 'concentric_cyl_musubi',
                          attachment = True,
                          validation = True,
                          val_method = 'difference',
                          val_ref_path = testcase_path+'tracking/ref_poisson_probe_p00000.res',
                          val_output_filename = 'tracking/poisson_probe_p00000.res',
                          ))
