Replace ReferenceTests with our own function (#164)

Project.toml

@@ -17,11 +17,11 @@ Parameters = "0.12"
 julia = "1"
 
 [extras]
+DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-ReferenceTests = "324d217c-45ce-50fc-942e-d289b448e8cf"
 SimpleMock = "a896ed2c-15a5-4479-b61d-a0e88e2a1d25"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Suppressor", "Random", "ReferenceTests", "SimpleMock", "Test"]
+test = ["DeepDiffs", "Random", "Suppressor", "SimpleMock", "Test"]

docs/src/developer.md

@@ -358,12 +358,9 @@ They should pass, and there will be new files in `test/fixtures`.
 Check them to make sure that they contain exactly what you would expect!
 
 For changes to existing plugins, update the plugin options appropriately in the "Wacky options" test set.
-Rather than using `Pkg.test` to run the tests in this case, use `include("test/runtests.jl")`.
+Failing tests  will give you the option to review and accept changes to the fixtures, updating the files automatically for you.
-Running the tests in an interactive session will give you the option to review and accept changes to the fixtures, updating the files automatically (see [ReferenceTests](https://github.com/Evizero/ReferenceTests.jl) for more details).
-Before you can do this, you'll need to add PkgTemplates' test requirements to your current environment.
 
 ### Updating "Show" Tests
 
 Depending on what you've changed, the tests in `test/show.jl` might fail.
-To fix those, you'll need to update the `expected` value to match what is actually displayed in a Julia REPL.
+To fix those, you'll need to update the `expected` value to match what is actually displayed in a Julia REPL (assuming that the new value is correct).
-The test error itself is usually not very helpful, so it's easiest to launch a REPL, evaluate `Template()`, and copy-paste the relevant parts back into the test file.

test/reference.jl

@@ -1,3 +1,4 @@
+const PROMPT = get(ENV, "PT_INTERACTIVE", "false") == "true" || !haskey(ENV, "CI")
 const STATIC_FILE = joinpath(@__DIR__, "fixtures", "static.txt")
 const STATIC_DOCUMENTER = [
     PackageSpec(; name="DocStringExtensions", version=v"0.8.1"),
@@ -6,6 +7,51 @@ const STATIC_DOCUMENTER = [
     PackageSpec(; name="Parsers", version=v"0.3.10"),
 ]
 
+function test_reference(reference, comparison)
+    if !isfile(comparison)
+        # If the comparison file doesn't yet exist, create it and pass the test.
+        @info "Creating new reference file $comparison"
+        copy_file(reference, comparison)
+        @test true
+        return
+    end
+
+    a = read(reference, String)
+    b = read(comparison, String)
+    if a == b
+        # If the files are equal, pass the test.
+        @test true
+        return
+    end
+
+    print_diff(a, b)
+    println("Reference and comparison files do not match (see above)")
+    println("Reference: $reference")
+    println("Comparison: $comparison")
+    update = false
+    if PROMPT
+        while true
+            println("Update reference file? [y/n]")
+            answer = lowercase(strip(readline()))
+            if startswith(answer, "y") || startswith(answer, "n")
+                # If the user chooses to update the reference file,
+                # replace its contents with the comparison file.
+                startswith(answer, "y") && copy_file(comparison, reference)
+                break
+            end
+        end
+    end
+
+    # Fail the test, but keep the output short
+    # because we've already showed the diff.
+    @test :reference == :comparison
+end
+
+function copy_file(src::AbstractString, dest::AbstractString)
+    mkpath(dirname(dest))
+    cp(src, dest; force=true)
+end
+
 PT.user_view(::Citation, ::Template, ::AbstractString) = Dict("MONTH" => 8, "YEAR" => 2019)
 PT.user_view(::License, ::Template, ::AbstractString) = Dict("YEAR" => 2019)
 
@@ -26,8 +72,8 @@ function test_all(pkg::AbstractString; kwargs...)
         PT.hasplugin(t, Documenter) && pin_documenter(joinpath(pkg_dir, "docs"))
         foreach(readlines(`git -C $pkg_dir ls-files`)) do f
             reference = joinpath(@__DIR__, "fixtures", pkg, f)
-            observed = read(joinpath(pkg_dir, f), String)
+            comparison = joinpath(pkg_dir, f)
-            @test_reference reference observed
+            test_reference(reference, comparison)
         end
     end
 end

test/runtests.jl

@@ -6,7 +6,7 @@ using Pkg: Pkg, PackageSpec, TOML
 using Random: Random, randstring
 using Test: @test, @testset, @test_logs, @test_throws
 
-using ReferenceTests: @test_reference
+using DeepDiffs: deepdiff
 using SimpleMock: mock
 using Suppressor: @suppress
 
@@ -36,6 +36,16 @@ function with_pkg(f::Function, t::Template, pkg::AbstractString=pkgname())
     end
 end
 
+function print_diff(a, b)
+    old = Base.have_color
+    @eval Base have_color = true
+    try
+        println(deepdiff(a, b))
+    finally
+        @eval Base have_color = $old
+    end
+end
+
 mktempdir() do dir
     Pkg.activate(dir)
     pushfirst!(DEPOT_PATH, dir)

test/show.jl

@@ -1,6 +1,15 @@
 const TEMPLATES_DIR = contractuser(PT.TEMPLATES_DIR)
 const LICENSES_DIR = joinpath(TEMPLATES_DIR, "licenses")
 
+function test_show(expected::AbstractString, observed::AbstractString)
+    if expected == observed
+        @test true
+    else
+        print_diff(expected, observed)
+        @test :expected == :observed
+    end
+end
+
 @testset "Show methods" begin
     @testset "Plugins" begin
         expected = """
@@ -9,7 +18,7 @@ const LICENSES_DIR = joinpath(TEMPLATES_DIR, "licenses")
               destination: "README.md"
               inline_badges: false
             """
-        @test sprint(show, MIME("text/plain"), Readme()) == rstrip(expected)
+        test_show(rstrip(expected), sprint(show, MIME("text/plain"), Readme()))
     end
 
     @testset "Template" begin
@@ -62,7 +71,7 @@ const LICENSES_DIR = joinpath(TEMPLATES_DIR, "licenses")
                   file: "$(joinpath(TEMPLATES_DIR, "test", "runtests.jl"))"
                   project: false
             """
-        @test sprint(show, MIME("text/plain"), tpl(; authors=USER)) == rstrip(expected)
+        test_show(rstrip(expected), sprint(show, MIME("text/plain"), tpl(; authors=USER)))
     end
 
     @testset "show as serialization" begin