Don't spawn a task per file to include, use a pool of tasks instead

Author: Drvi

src/ReTestItems.jl

@@ -802,7 +802,84 @@ function _is_subproject(dir, current_projectfile)
     return true
 end
 
-# for each directory, kick off a recursive test-finding task
+const _hidden_re = r"\.\w"
+
+# Traverses the directory tree starting at `project_root` and populates `dir_nodes` with
+# `DirNode`s and `FileNode`s for each directory and test file found. Filters out non-eligible
+# paths.
+function walkdir_task(walkdir_channel::Channel{Tuple{String,FileNode}}, project_root::String, root_node, dir_nodes, subproject_root::Union{String,Nothing}, ti_filter, paths, projectfile, report, verbose_results)
+    try
+        # Since test items don't store paths to their test setups, we need to traverse the
+        # whole project, not just the requested paths.
+        stack = [project_root]
+        while !isempty(stack)
+            root = pop!(stack)
+            rel_root = nestedrelpath(root, project_root)
+            dir_node = DirNode(rel_root; report, verbose=verbose_results)
+            dir_nodes[rel_root] = dir_node
+            push!(get(dir_nodes, dirname(rel_root), root_node), dir_node)
+            for file in readdir(root)
+                startswith(file, _hidden_re) && continue # skip hidden files/directories
+                full_path = joinpath(root, file)
+                if isdir(full_path)
+                    if subproject_root !== nothing && startswith(full_path, subproject_root)
+                        @debugv 1 "Skipping files in `$root` in subproject `$subproject_root`"
+                        continue
+                    elseif _is_subproject(root, projectfile)
+                        subproject_root = root
+                        continue
+                    end
+                    push!(stack, full_path)
+                else
+                    # We filter here, rather than the testitem level, to make sure we don't
+                    # `include` a file that isn't supposed to be a test-file at all, e.g. its
+                    # not on a path the user requested but it happens to have a test-file suffix.
+                    # We always include testsetup-files so users don't need to request them,
+                    # even if they're not in a requested path, e.g. they are a level up in the
+                    # directory tree. The testsetup-file suffix is hopefully specific enough
+                    # to ReTestItems that this doesn't lead to `include`ing unexpected files.
+                    if !(is_testsetup_file(full_path) || (is_test_file(full_path) && is_requested(full_path, paths)))
+                        continue
+                    end
+                    rel_full_path = nestedrelpath(full_path, project_root)
+                    file_node = FileNode(rel_full_path, ti_filter; report, verbose=verbose_results)
+                    push!(dir_node, file_node)
+                    put!(walkdir_channel, (full_path, file_node))
+                end
+            end
+        end
+        close(walkdir_channel)
+    catch err
+        close(walkdir_channel, err)
+        rethrow(err)
+    end
+    return nothing
+end
+
+# Parses and evals files found by the `walkdir_task`. During macro expansion of `@testitem`
+# test items are push!d onto the FileNode stored in task local storage as `:__RE_TEST_ITEMS__`.
+function include_task(walkdir_channel, setup_channel, project_root, ti_filter)
+    try
+        testitem_names = Set{String}() # to enforce that names in the same file are unique
+        task_local_storage(:__RE_TEST_RUNNING__, true) do
+            task_local_storage(:__RE_TEST_PROJECT__, project_root) do
+                task_local_storage(:__RE_TEST_SETUPS__, setup_channel) do
+                    for (file_path, file_node) in walkdir_channel
+                        @debugv 1 "Including test items from file `$(file_path)`"
+                        task_local_storage(:__RE_TEST_ITEMS__, (file_node, empty!(testitem_names))) do
+                            Base.include(ti_filter, Main, file_path)
+                        end
+                    end
+                end
+            end
+        end
+    catch err
+        close(walkdir_channel, err)
+        rethrow(err)
+    end
+end
+
+# Find test items using a pool of tasks to include files in parallel.
 # Returns (testitems::TestItems, setups::Dict{Symbol,TestSetup})
 # Assumes `isdir(project_root)`, which is guaranteed by `_runtests`.
 function include_testfiles!(project_name, projectfile, paths, ti_filter::TestItemFilter, verbose_results::Bool, report::Bool)
@@ -823,51 +900,18 @@ function include_testfiles!(project_name, projectfile, paths, ti_filter::TestIte
         end
         return setups
     end
-    hidden_re = r"\.\w"
-    @sync for (root, d, files) in Base.walkdir(project_root)
-        if subproject_root !== nothing && startswith(root, subproject_root)
-            @debugv 1 "Skipping files in `$root` in subproject `$subproject_root`"
-            continue
-        elseif _is_subproject(root, projectfile)
-            subproject_root = root
-            continue
-        end
-        rpath = nestedrelpath(root, project_root)
-        startswith(rpath, hidden_re) && continue # skip hidden directories
-        dir_node = DirNode(rpath; report, verbose=verbose_results)
-        dir_nodes[rpath] = dir_node
-        push!(get(dir_nodes, dirname(rpath), root_node), dir_node)
-        for file in files
-            startswith(file, hidden_re) && continue # skip hidden files
-            filepath = joinpath(root, file)
-            # We filter here, rather than the testitem level, to make sure we don't
-            # `include` a file that isn't supposed to be a test-file at all, e.g. its
-            # not on a path the user requested but it happens to have a test-file suffix.
-            # We always include testsetup-files so users don't need to request them,
-            # even if they're not in a requested path, e.g. they are a level up in the
-            # directory tree. The testsetup-file suffix is hopefully specific enough
-            # to ReTestItems that this doesn't lead to `include`ing unexpected files.
-            if !(is_testsetup_file(filepath) || (is_test_file(filepath) && is_requested(filepath, paths)))
-                continue
-            end
-            fpath = nestedrelpath(filepath, project_root)
-            file_node = FileNode(fpath, ti_filter; report, verbose=verbose_results)
-            testitem_names = Set{String}() # to enforce that names in the same file are unique
-            push!(dir_node, file_node)
-            @debugv 1 "Including test items from file `$filepath`"
-            @spawn begin
-                task_local_storage(:__RE_TEST_RUNNING__, true) do
-                    task_local_storage(:__RE_TEST_ITEMS__, ($file_node, $testitem_names)) do
-                        task_local_storage(:__RE_TEST_PROJECT__, $(project_root)) do
-                            task_local_storage(:__RE_TEST_SETUPS__, $setup_channel) do
-                                Base.include($ti_filter, Main, $filepath)
-                            end
-                        end
-                    end
-                end
-            end
+
+    walkdir_channel = Channel{Tuple{String, FileNode}}(1024)
+    @sync begin
+        @spawn walkdir_task(
+            $walkdir_channel, $project_root, $root_node, $dir_nodes, $subproject_root,
+            $ti_filter, $paths, $projectfile, $report, $verbose_results
+        )
+        for _ in 1:clamp(2*nthreads(), 1, 16)
+            @spawn include_task($walkdir_channel, $setup_channel, $project_root, $ti_filter)
         end
     end
+
     @debugv 2 "Finished including files"
     # finished including all test files, so finalize our graph
     # prune empty directories/files