diff options
| -rw-r--r-- | src/Eval.hs | 74 | ||||
| -rw-r--r-- | test/asset/repo/checkout.yaml | 93 | ||||
| -rw-r--r-- | test/asset/run/dependencies.yaml | 27 | ||||
| -rw-r--r-- | test/script/repo.et | 176 | ||||
| -rw-r--r-- | test/script/run.et | 44 |
5 files changed, 375 insertions, 39 deletions
diff --git a/src/Eval.hs b/src/Eval.hs index b73f0f3..1b0d7dd 100644 --- a/src/Eval.hs +++ b/src/Eval.hs @@ -54,6 +54,29 @@ commonPrefix :: Eq a => [ a ] -> [ a ] -> [ a ] commonPrefix (x : xs) (y : ys) | x == y = x : commonPrefix xs ys commonPrefix _ _ = [] +checkIfAlreadyHasDefaultRepoId :: Eval Bool +checkIfAlreadyHasDefaultRepoId = do + asks (any isDefaultRepoId . eiCurrentIdRev) + where + isDefaultRepoId (JobIdName _) = False + isDefaultRepoId (JobIdCommit rname _) = isNothing rname + isDefaultRepoId (JobIdTree rname _ _) = isNothing rname + +collectJobSetRepos :: [ ( Maybe RepoName, Tree ) ] -> DeclaredJobSet -> Eval [ ( Maybe RepoName, Tree ) ] +collectJobSetRepos revisionOverrides dset = do + jobs <- either (throwError . OtherEvalError . T.pack) return $ jobsetJobsEither dset + let someJobUsesDefaultRepo = any (any (isNothing . jcRepo) . jobCheckout) jobs + repos = + (if someJobUsesDefaultRepo then (Nothing :) else id) $ + map (Just . repoName) $ maybe [] configRepos $ jobsetConfig dset + forM repos $ \rname -> do + case lookup rname revisionOverrides of + Just tree -> return ( rname, tree ) + Nothing -> do + repo <- evalRepo rname + tree <- getCommitTree =<< readCommit repo "HEAD" + return ( rname, tree ) + collectOtherRepos :: DeclaredJobSet -> DeclaredJob -> Eval [ ( Maybe ( RepoName, Maybe Text ), FilePath ) ] collectOtherRepos dset decl = do jobs <- either (throwError . OtherEvalError . T.pack) return $ jobsetJobsEither dset @@ -69,10 +92,7 @@ collectOtherRepos dset decl = do job <- maybe (throwError $ OtherEvalError $ "job ‘" <> textJobName name <> "’ not found") return . find ((name ==) . jobName) $ jobs return $ jobCheckout job - let isDefaultRepoId (JobIdName _) = False - isDefaultRepoId (JobIdCommit rname _) = isNothing rname - isDefaultRepoId (JobIdTree rname _ _) = isNothing rname - alreadyHasDefaultRepoId <- asks (any isDefaultRepoId . eiCurrentIdRev) + alreadyHasDefaultRepoId <- checkIfAlreadyHasDefaultRepoId let checkouts = (if alreadyHasDefaultRepoId then filter (isJust . jcRepo) else id) $ concat dependencyRepos @@ -132,6 +152,13 @@ evalJob revisionOverrides dset decl = do evalJobSet :: [ ( Maybe RepoName, Tree ) ] -> DeclaredJobSet -> Eval JobSet evalJobSet revisionOverrides decl = do EvalInput {..} <- ask + repos <- collectJobSetRepos revisionOverrides decl + alreadyHasDefaultRepoId <- checkIfAlreadyHasDefaultRepoId + let addedRepoIds = + map (\( mbname, tree ) -> JobIdTree mbname "" (treeId tree)) $ + (if alreadyHasDefaultRepoId then filter (isJust . fst) else id) $ + repos + jobs <- fmap (fmap (map fst)) $ either (return . Left) (handleToEither . mapM (evalJob revisionOverrides decl)) $ jobsetJobsEither decl @@ -141,7 +168,7 @@ evalJobSet revisionOverrides decl = do Right declEval -> catMaybes $ map (\jid -> jobId . snd <$> find ((jid ==) . jobId . fst) declEval) $ jobsetExplicitlyRequested decl return JobSet - { jobsetId = JobSetId $ reverse $ eiCurrentIdRev + { jobsetId = JobSetId $ reverse $ reverse addedRepoIds ++ eiCurrentIdRev , jobsetConfig = jobsetConfig decl , jobsetCommit = jobsetCommit decl , jobsetExplicitlyRequested = explicit @@ -162,7 +189,13 @@ evalRepo (Just name) = asks (lookup name . eiOtherRepos) >>= \case canonicalJobName :: [ Text ] -> Config -> Maybe Tree -> Eval JobSet canonicalJobName (r : rs) config mbDefaultRepo = do let name = JobName r - dset = JobSet () (Just config) Nothing [] $ Right $ configJobs config + dset = JobSet + { jobsetId = () + , jobsetConfig = Just config + , jobsetCommit = Nothing + , jobsetExplicitlyRequested = [] + , jobsetJobsEither = Right $ configJobs config + } case find ((name ==) . jobName) (configJobs config) of Just djob -> do otherRepos <- collectOtherRepos dset djob @@ -177,14 +210,8 @@ canonicalJobName (r : rs) config mbDefaultRepo = do case rs' of (r' : _) -> throwError $ OtherEvalError $ "unexpected job ref part ‘" <> r' <> "’" _ -> return () - ( job, sid ) <- evalJob (maybe id ((:) . ( Nothing, )) mbDefaultRepo $ overrides) dset djob - return JobSet - { jobsetId = sid - , jobsetConfig = Just config - , jobsetCommit = Nothing - , jobsetExplicitlyRequested = [] - , jobsetJobsEither = Right [ job ] - } + eset <- evalJobSet (maybe id ((:) . ( Nothing, )) mbDefaultRepo $ overrides) dset + return eset { jobsetJobsEither = fmap (filter ((name ==) . jobName)) $ jobsetJobsEither eset } Nothing -> throwError $ OtherEvalError $ "job ‘" <> r <> "’ not found" canonicalJobName [] _ _ = throwError $ OtherEvalError "expected job name" @@ -266,20 +293,21 @@ fillInDependencies :: JobSet -> Eval JobSet fillInDependencies jset = do ( dset, idRev, otherRepos ) <- local (\ei -> ei { eiCurrentIdRev = [] }) $ do loadJobSetById (jobsetId jset) + eset <- local (\ei -> ei { eiCurrentIdRev = idRev }) $ do + evalJobSet otherRepos dset origJobs <- either (throwError . OtherEvalError . T.pack) return $ jobsetJobsEither jset - declJobs <- either (throwError . OtherEvalError . T.pack) return $ jobsetJobsEither dset - deps <- gather declJobs S.empty (map jobName origJobs) + allJobs <- either (throwError . OtherEvalError . T.pack) return $ jobsetJobsEither eset + deps <- gather allJobs S.empty (map jobName origJobs) - jobs <- local (\ei -> ei { eiCurrentIdRev = idRev }) $ do - fmap catMaybes $ forM declJobs $ \djob -> if - | Just job <- find ((jobName djob ==) . jobName) origJobs - -> return (Just job) + let jobs = catMaybes $ flip map allJobs $ \ejob -> if + | Just job <- find ((jobName ejob ==) . jobName) origJobs + -> Just job - | jobName djob `S.member` deps - -> Just . fst <$> evalJob otherRepos dset djob + | jobName ejob `S.member` deps + -> Just ejob | otherwise - -> return Nothing + -> Nothing return $ jset { jobsetJobsEither = Right jobs } where diff --git a/test/asset/repo/checkout.yaml b/test/asset/repo/checkout.yaml new file mode 100644 index 0000000..f1bcbc8 --- /dev/null +++ b/test/asset/repo/checkout.yaml @@ -0,0 +1,93 @@ +repo r1: +repo r2: +repo r3: + +job checkout_subtree: + checkout: + - repo: r1 + dest: d1 + + - repo: r2 + subtree: s1 + dest: d2_s1 + + shell: + - find \! -name 'list?' -type f > list1 + + artifact out: + path: list1 + + +job checkout_common_root: + checkout: + - repo: r1 + dest: d1 + + - repo: r2 + subtree: s1 + dest: d2_s1 + + - repo: r2 + subtree: s2 + dest: d2_s2 + + shell: + - find \! -name 'list?' -type f > list2 + + artifact out: + path: list2 + + +job checkout_common_subtree: + checkout: + - repo: r3 + subtree: a/b/s1 + dest: d3_s1 + + - repo: r3 + subtree: a/b/c/s2 + dest: d3_s2 + + shell: + - find \! -name 'list?' -type f > list3 + + artifact out: + path: list3 + + +job checkout_common_subtree2: + checkout: + - repo: r1 + dest: d1 + + - repo: r2 + subtree: s1 + dest: d2_s1 + + - repo: r3 + subtree: a/b/c/s2 + dest: d3_s2 + + - repo: r3 + subtree: a/b/c/s3 + dest: d3_s3 + + shell: + - find \! -name 'list?' -type f > list4 + + artifact out: + path: list4 + + +job combined_subtree: + uses: + - checkout_common_root.out + - checkout_common_subtree2.out + + +job combined_all: + uses: + - checkout_subtree.out + - checkout_common_root.out + - checkout_common_subtree.out + - checkout_common_subtree2.out diff --git a/test/asset/run/dependencies.yaml b/test/asset/run/dependencies.yaml index 7452b5a..13d5e63 100644 --- a/test/asset/run/dependencies.yaml +++ b/test/asset/run/dependencies.yaml @@ -1,3 +1,5 @@ +repo other: + job first: shell: - touch x @@ -29,6 +31,11 @@ job third: job fourth: + checkout: + - dest: a + - repo: other + dest: b + uses: - second.out @@ -40,16 +47,16 @@ job fourth: job fifth: - uses: - - third.out - - fourth.out + uses: + - third.out + - fourth.out - shell: - - mv z z2 - - mv w w2 + shell: + - mv z z2 + - mv w w2 - artifact out1: - path: z2 + artifact out1: + path: z2 - artifact out2: - path: w2 + artifact out2: + path: w2 diff --git a/test/script/repo.et b/test/script/repo.et index d93f700..aeeeb6a 100644 --- a/test/script/repo.et +++ b/test/script/repo.et @@ -1,3 +1,11 @@ +module repo + +import common + +asset scripts: + path: ../asset/repo + + test RepoSubtree: node n shell on n as git_init: @@ -87,3 +95,171 @@ test RepoSubtree: expect from p /msg $sub2/ expect from p /path (.*)/ capture path guard (path == "first/second") + + +test CheckoutSubtree: + node n + shell on n as git_init: + mkdir -p main + git -C main -c init.defaultBranch=master init -q + cp "${scripts.path}/checkout.yaml" main/minici.yaml + git -C main add minici.yaml + git -C main -c user.name=test -c user.email=test commit -q --allow-empty -m 'initial commit' + git -C main rev-parse HEAD + git -C main rev-parse HEAD^{tree} + + mkdir -p dir_r1 + git -C dir_r1 -c init.defaultBranch=master init -q + git -C dir_r1 -c user.name=test -c user.email=test commit -q --allow-empty -m 'initial commit' + touch dir_r1/file + git -C dir_r1 add . + git -C dir_r1 -c user.name=test -c user.email=test commit -q --allow-empty -m 'commit r1' + git -C dir_r1 rev-parse HEAD + git -C dir_r1 rev-parse HEAD^{tree} + + mkdir -p dir_r2 + git -C dir_r2 -c init.defaultBranch=master init -q + git -C dir_r2 -c user.name=test -c user.email=test commit -q --allow-empty -m 'initial commit' + mkdir -p dir_r2/s1 + mkdir -p dir_r2/s2 + touch dir_r2/s1/file1 + touch dir_r2/s2/file2 + git -C dir_r2 add . + git -C dir_r2 -c user.name=test -c user.email=test commit -q --allow-empty -m 'commit r2' + git -C dir_r2 rev-parse HEAD + git -C dir_r2 rev-parse HEAD^{tree} + git -C dir_r2 rev-parse HEAD^{tree}:s1 + git -C dir_r2 rev-parse HEAD^{tree}:s2 + + mkdir -p dir_r3 + git -C dir_r3 -c init.defaultBranch=master init -q + git -C dir_r3 -c user.name=test -c user.email=test commit -q --allow-empty -m 'initial commit' + mkdir -p dir_r3/a/b/s1 + mkdir -p dir_r3/a/b/c/s2 + mkdir -p dir_r3/a/b/c/s3 + touch dir_r3/a/b/s1/file1 + touch dir_r3/a/b/c/s2/file2 + touch dir_r3/a/b/c/s3/file3 + git -C dir_r3 add . + git -C dir_r3 -c user.name=test -c user.email=test commit -q --allow-empty -m 'commit r3' + git -C dir_r3 rev-parse HEAD + git -C dir_r3 rev-parse HEAD^{tree} + git -C dir_r3 rev-parse HEAD^{tree}:a/b + git -C dir_r3 rev-parse HEAD^{tree}:a/b/s1 + git -C dir_r3 rev-parse HEAD^{tree}:a/b/c + git -C dir_r3 rev-parse HEAD^{tree}:a/b/c/s2 + git -C dir_r3 rev-parse HEAD^{tree}:a/b/c/s3 + + expect /([0-9a-f]+)/ from git_init capture mc + expect /([0-9a-f]+)/ from git_init capture mt + + expect /([0-9a-f]+)/ from git_init capture r1c + expect /([0-9a-f]+)/ from git_init capture r1t + + expect /([0-9a-f]+)/ from git_init capture r2c + expect /([0-9a-f]+)/ from git_init capture r2t + expect /([0-9a-f]+)/ from git_init capture r2s1 + expect /([0-9a-f]+)/ from git_init capture r2s2 + + expect /([0-9a-f]+)/ from git_init capture r3c + expect /([0-9a-f]+)/ from git_init capture r3t + expect /([0-9a-f]+)/ from git_init capture r3ab + expect /([0-9a-f]+)/ from git_init capture r3abs1 + expect /([0-9a-f]+)/ from git_init capture r3abc + expect /([0-9a-f]+)/ from git_init capture r3abcs2 + expect /([0-9a-f]+)/ from git_init capture r3abcs3 + + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "checkout_subtree" ] + expect_success from p of "$mt.checkout_subtree.$r1t.$r2s1" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "$mt.checkout_subtree.$r1t.$r2s1" ] + expect_success from p of "$mt.checkout_subtree.$r1t.$r2s1" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "extract", "$mt.checkout_subtree.$r1t.$r2s1.out", "list" ] + local: + shell on n as list: + cat list + rm list + echo DONE + expect from list: + /\.\/d1\/file/ + /\.\/d2_s1\/file1/ + /(.*)/ capture done + guard (done == "DONE") + + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "checkout_common_root" ] + expect_success from p of "$mt.checkout_common_root.$r1t.$r2t" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "$mt.checkout_common_root.$r1t.$r2t" ] + expect_success from p of "$mt.checkout_common_root.$r1t.$r2t" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "extract", "$mt.checkout_common_root.$r1t.$r2t.out", "list" ] + local: + shell on n as list: + cat list + rm list + echo DONE + expect from list: + /\.\/d1\/file/ + /\.\/d2_s1\/file1/ + /\.\/d2_s2\/file2/ + /(.*)/ capture done + guard (done == "DONE") + + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "checkout_common_subtree" ] + expect_success from p of "$mt.checkout_common_subtree.$r3ab" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "$mt.checkout_common_subtree.$r3ab" ] + expect_success from p of "$mt.checkout_common_subtree.$r3ab" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "extract", "$mt.checkout_common_subtree.$r3ab.out", "list" ] + local: + shell on n as list: + cat list + rm list + echo DONE + expect from list: + /\.\/d3_s1\/file1/ + /\.\/d3_s2\/file2/ + /(.*)/ capture done + guard (done == "DONE") + + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "checkout_common_subtree2" ] + expect_success from p of "$mt.checkout_common_subtree2.$r1t.$r2s1.$r3abc" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "run", "--rerun-all", "$mt.checkout_common_subtree2.$r1t.$r2s1.$r3abc" ] + expect_success from p of "$mt.checkout_common_subtree2.$r1t.$r2s1.$r3abc" + expect /(.*)/ from p capture done + guard (done == "run-finish") + local: + spawn on n as p args [ "--storage=minici", "--repo=r1:./dir_r1", "--repo=r2:./dir_r2", "--repo=r3:./dir_r3", "./main", "extract", "$mt.checkout_common_subtree2.$r1t.$r2s1.$r3abc.out", "list" ] + local: + shell on n as list: + cat list + rm list + echo DONE + expect from list: + /\.\/d1\/file/ + /\.\/d2_s1\/file1/ + /\.\/d3_s2\/file2/ + /\.\/d3_s3\/file3/ + /(.*)/ capture done + guard (done == "DONE") diff --git a/test/script/run.et b/test/script/run.et index 124ae37..f00a4a7 100644 --- a/test/script/run.et +++ b/test/script/run.et @@ -260,31 +260,63 @@ test RunExplicitDependentJob: git -C main -c user.name=test -c user.email=test commit -q -m 'commit' git -C main rev-parse HEAD^{tree} + mkdir -p other + git -C other -c init.defaultBranch=master init -q + + mkdir -p other/subdir + + touch other/subdir/a + git -C other add subdir + git -C other -c user.name=test -c user.email=test commit -q -m 'commit' + git -C other rev-parse HEAD^{tree} + + touch other/subdir/b + git -C other add subdir + git -C other -c user.name=test -c user.email=test commit -q -m 'commit' + git -C other rev-parse HEAD^{tree} + + rm other/subdir/a + rm other/subdir/b + touch other/subdir/c + git -C other add subdir + git -C other -c user.name=test -c user.email=test commit -q -m 'commit' + git -C other rev-parse HEAD^{tree} + + touch other/subdir/d + git -C other add subdir + git -C other -c user.name=test -c user.email=test commit -q -m 'commit' + git -C other rev-parse HEAD^{tree} + expect /([0-9a-f]+)/ from git_init capture c1 expect /([0-9a-f]+)/ from git_init capture t1 expect /([0-9a-f]+)/ from git_init capture t2 expect /([0-9a-f]+)/ from git_init capture t3 expect /([0-9a-f]+)/ from git_init capture t4 + expect /([0-9a-f]+)/ from git_init capture o1 + expect /([0-9a-f]+)/ from git_init capture o2 + expect /([0-9a-f]+)/ from git_init capture o3 + expect /([0-9a-f]+)/ from git_init capture o4 + local: - spawn on n as p args [ "./main", "run", "$c1.first", "$t2.first", "$t3.fourth", "$c1.fifth", "$c1.fourth", "$c1.third", "$c1.second", "$t4.fifth" ] + spawn on n as p args [ "--repo=other:./other", "./main", "run", "$c1.first", "$t2.first", "$t3.fourth.$o3", "$c1.fifth.$o1", "$c1.fourth.$o1", "$c1.third", "$c1.second", "$t4.fifth.$o4" ] expect_success from p of "$t1.first" expect_success from p of "$t1.second" expect_success from p of "$t1.third" - expect_success from p of "$t1.fourth" - expect_success from p of "$t1.fifth" + expect_success from p of "$t1.fourth.$o1" + expect_success from p of "$t1.fifth.$o1" expect_success from p of "$t2.first" expect_success from p of "$t3.first" expect_success from p of "$t3.second" - expect_success from p of "$t3.fourth" + expect_success from p of "$t3.fourth.$o3" expect_success from p of "$t4.first" expect_success from p of "$t4.second" expect_success from p of "$t4.third" - expect_success from p of "$t4.fourth" - expect_success from p of "$t4.fifth" + expect_success from p of "$t4.fourth.$o4" + expect_success from p of "$t4.fifth.$o4" flush from p matching /note .*/ flush from p matching /job-duplicate .*/ |