Add datasets from Embrace Uncertainty and recent SMLPs

Project.toml

@@ -1,20 +1,26 @@
 name = "MixedModelsDatasets"
 uuid = "7e9fb7ac-9f67-43bf-b2c8-96ba0796cbb6"
 authors = ["Phillip Alday <me@phillipalday.com>", "Douglas Bates <dmbates@gmail.com>"]
-version = "0.1.2"
+version = "0.2.0"
 
 [deps]
 Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
-Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
-LazyArtifacts = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
+DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 
 [compat]
 Aqua = "0.8"
 Arrow = "1, 2"
-Artifacts = "1"
-LazyArtifacts = "1"
-Test = "0.0, 1"
-julia = "1.6"
+DelimitedFiles = "1"
+Downloads = "1"
+ProgressMeter = "1"
+SHA = "0.7"
+Scratch = "1"
+Test = "1"
+julia = "1.10"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"

_typos.toml

@@ -23,6 +23,7 @@ verbagg = "verbagg"
 grouseticks = "grouseticks"
 dyestuff = "dyestuff"
 covariate = "covariate"
+frstLang = "frstLang"
 
 [default.extend-words]
 Lik = "Lik"

src/MixedModelsDatasets.jl

@@ -1,13 +1,46 @@
 module MixedModelsDatasets
 
 using Arrow
-using Artifacts
-using LazyArtifacts
+using DelimitedFiles
+using Downloads
+using ProgressMeter
+using Scratch
+using SHA
 
 export dataset, datasets
 
-_testdata() = artifact"TestData"
-cacheddatasets = Dict{String,Arrow.Table}()
+const CACHE = Ref("")
+const CACHED_DATASETS = Dict{String,String}()
+const DATASETS = Dict{String,NamedTuple}()
+
+"""
+    __init__()
+
+Initialize the package scratchspace and populate the internal registry of
+available datasets by parsing `datasets.csv`.
+"""
+function __init__()
+    CACHE[] = @get_scratch!("data")
+
+    csv = joinpath(@__DIR__, "datasets.csv")
+    data, headers = readdlm(csv, ',', String;
+                            header=true, skipblanks=true, comments=true)
+    names = Symbol.(Tuple(headers))
+    rows = NamedTuple{names}.(eachrow(data))
+    for row in rows
+        DATASETS[row.dsname] = row
+    end
+
+    return nothing
+end
+
+"""
+    clear_scratchspaces!()
+
+Remove all cached datasets by clearing the package's scratchspace, forcing
+subsequent downloads to be fetched anew.
+"""
+clear_scratchspaces!() = Scratch.clear_scratchspaces!(@__MODULE__)
 
 """
     dataset(nm)
@@ -18,13 +51,8 @@ Return, as an `Arrow.Table`, the test data set named `nm`, which can be a `Strin
     Dataset names are case insensitive: internally all names are normalized to lowercase.
 """
 function dataset(nm::AbstractString)
-    get!(cacheddatasets, nm) do
-        path = joinpath(_testdata(), lowercase(nm) * ".arrow")
-        if !isfile(path)
-            throw(ArgumentError("Dataset \"$nm\" is not available.\nUse MixedModels.datasets() for available names."))
-        end
-        return Arrow.Table(path)
-    end
+    path = _download(nm)
+    return Arrow.Table(path)
 end
 dataset(nm::Symbol) = dataset(string(nm))
 
@@ -33,9 +61,58 @@ dataset(nm::Symbol) = dataset(string(nm))
 
 Return a vector of names of the available test data sets
 """
-function datasets()
-    return first.(Base.Filesystem.splitext.(filter(endswith(".arrow"),
-                                                   readdir(_testdata()))))
+function datasets(; downloaded=false)
+    ds = collect(keys(DATASETS))
+    if downloaded
+        intersect!(ds, keys(CACHED_DATASETS))
+    end
+    return sort!(ds)
 end
 
+"""
+    _download(nm::AbstractString; info=true)
+
+Return the local cache path for dataset `nm`, downloading it (and verifying
+its SHA-256 checksum) if it is not already cached or the cached file's
+checksum no longer matches. Set `info=false` to suppress the `@info` log
+messages.
+"""
+function _download(nm::AbstractString; info=true)
+    nm = lowercase(nm)
+    return get!(CACHED_DATASETS, nm) do
+        nm in keys(DATASETS) ||
+            throw(ArgumentError("Dataset \"$nm\" is not available.\nUse MixedModels.datasets() for available names."))
+        path = joinpath(CACHE[], nm * ".arrow")
+        ds = DATASETS[nm]
+        if !isfile(path) || ds.sha2 != bytes2hex(open(sha2_256, path))
+            info && @info "Downloading dataset..."
+            url = string("https://osf.io/", ds.filename, "/download?version=", ds.version)
+            Downloads.download(url, path)
+            ds.sha2 == bytes2hex(open(sha2_256, path)) ||
+                error("Downloaded file failed checksum verification.")
+            info && @info "done"
+        end
+
+        return path
+    end
+end
+
+"""
+    download_all()
+
+Download every available dataset into the local cache, displaying a progress
+bar as they are fetched.
+"""
+function download_all()
+    @showprogress asyncmap(x -> _download(x; info=false), collect(keys(DATASETS)))
+    return nothing
+end
+
+# function hex(row)
+#        Downloads.download(
+#                           string("https://osf.io/", row.filename, "/download?version=", row.version),
+#                            joinpath(CACHE[], lowercase(row.dsname) * ".arrow"))
+#        bytes2hex(open(sha2_256,joinpath(CACHE[], lowercase(row.dsname) * ".arrow")))
+# end
+
 end # module MixedModelsDatasets

src/datasets.csv

@@ -0,0 +1,29 @@
+dsname,project,filename,version,sha2
+box,d48ce,tkxnh,1,4ac2038735c5286ca0d6a706b4feef5b34bd93560bc4cabc7223addf2366e4c0
+elstongrizzle,d48ce,5vrbw,1,f33e08ad5a91ab5dd9889953dbb97cc52a299fdac9db8b37ec4f87ad2dacadbd
+oxboys,d48ce,cz6g3,1,079f46e404e43c1848a65d3a4e5b6a14b2cb17565c77819d7ee3effe72f5ebd0
+elp_ldt_item,d48ce,c6gxd,1,f851910e1435659ca662ad49cfb8deb6b7cf287e4ce4969103dba11b32ab2e6c
+elp_ldt_subj,d48ce,rqenu,2,d9c88915681b64fc9db975f9bb2d6f402058fee5cb35887f9de7d07776efdd56
+elp_ldt_trial,d48ce,3evhy,2,57a83679f8f458b1d9bb56e05099a556ffdaf15af67464e9ee608c844fc4fa9c
+fggk21,d48ce,vwecy,1,0fa959f095f8b92135496b6f8c8a8b5a3e896e8875f0ba6928bd074559d8a796
+kkl15,d48ce,p8cea,2,90d7bb137c8613d7a15c8597c461aee7c7cb0f0989a07c80fc93e1fbe2e5c156
+kwdyz11,d48ce,4cv52,3,2fa23aa8aa25e1adb10183c8d29646ae0d19d6baef9d711c9906f7fa1b225571
+cbpp,a94tr,dywvu,1,f528a88ae8427ab3b96b5e1ef1f4f143fc47c17780d6e30afb75e520e8bc0091
+contra,a94tr,n7jxd,1,5994ebe24bd1a3faa5e578a9eea9bb4737ad4025022e75d6b19bffd66e800152
+d3,a94tr,m6vjt,1,878a141a5621d906c352e0aa3a29032653919446f5a541c5ecc39b247f4a19ff
+dyestuff,a94tr,9vztj,1,e91a491547ef9e86d657370c0195d13047e3e5b06ac709f2aa89c6780ad24ab7
+dyestuff2,a94tr,sr5v4,1,59857f679232fa8d6eea15e877ebf1b691e5aea1925dffdff02ff7cc40c208e0
+grouseticks,a94tr,9jhnp,1,8041ecdfd62dd65d53da1f835d8b94fc78c5b78353d4dc72a1548760b5f9b152
+insteval,a94tr,fbdxa,1,1f46ae62a5e5519b6774392c1351629cc750b135981d20953711618105866878
+kb07,a94tr,xwh49,1,86549da5fdfb86b8562bc05d5ace44890cfd3673bb094e9a9f7bac6588cc2a0d
+machines,a94tr,uznmg,1,0a3a1adf7bd3b970da0a82a2daa6d6e76368c292d808ae0c8056e03b35a3b7b9
+ml1m,a94tr,fubnw,1,5316cd17af885accd741ccd7f4d8f0b597f3b6b46c5769e57a7bde31ca77956f
+mmec,a94tr,5cxrm,1,d63f4b75bd0705d2316e176b189fda8e5216e2cb72d1aa550ef010371a1a07a9
+mrk17_exp1,a94tr,mb3fj,1,2d70546bcb96993c4642a3962b174cbfe2627a41eeb3952c6f0d7ad4b8103a7b
+oxide,a94tr,zh5gj,1,b7ccdb74baf8662719b1621fd6912e551a747cdaba6c0858576ad1418ff80d4c
+pastes,a94tr,5cxd6,1,8130b80fd9be615f8da3a21f3b5c412c05f72350e40262869068c4648d80931f
+penicillin,a94tr,nk3tu,1,499357c35cc171f14fa38d373c5d40cbf40f82ede333f5a5ad3a1c9f1d027381
+sleepstudy,a94tr,cq5xr,1,6f607c260a88fc2e8960944fa65a0458661f24c1f1bac64eba98983232e29549
+verbagg,a94tr,nmy3f,1,a447d8d4cc0c737d541d47fc73f73d02d044106a207a82be5f33923756745331
+ml32_movies,a94tr,8d93r,1,e09894d3de1b7d74e22973bcc768a714b6254fe5f578fb01f5f8a92953082cef
+ml32_ratings,a94tr,dn2qk,1,27811b9c9ca44ae193a708e43d3fefa00206b8a745f7b55eda50ed8c56a2bbbb

test/runtests.jl

@@ -8,7 +8,7 @@ using Test
 end
 
 @testset "datasets" begin
-    @test length(datasets()) == 17
+    @test length(datasets()) == 28
 
     @testset "$(ds) loadable" for ds in datasets()
         @test dataset(ds) isa Arrow.Table