Add JLD2 results download format

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.8.2"
 manifest_format = "2.0"
-project_hash = "c29e8447aeee171858dde0b54605eebd985a214b"
+project_hash = "c18c5eaab58f1afb699d93a8a6f512a73acf8b7c"
 
 [[deps.ArgParse]]
 deps = ["Logging", "TextWrap"]
@@ -140,6 +140,12 @@ git-tree-sha1 = "0fa3b52a04a4e210aeb1626def9c90df3ae65268"
 uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615"
 version = "1.1.0"
 
+[[deps.FileIO]]
+deps = ["Pkg", "Requires", "UUIDs"]
+git-tree-sha1 = "94f5101b96d2d968ace56f7f2db19d0a5f592e28"
+uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+version = "1.15.0"
+
 [[deps.FilePathsBase]]
 deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"]
 git-tree-sha1 = "e27c4ebe80e8699540f2d6c805cc12203b614f12"
@@ -230,6 +236,12 @@ git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
 uuid = "82899510-4779-5014-852e-03e436cf321d"
 version = "1.0.0"
 
+[[deps.JLD2]]
+deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "Pkg", "Printf", "Reexport", "TranscodingStreams", "UUIDs"]
+git-tree-sha1 = "1c3ff7416cb727ebf4bab0491a56a296d7b8cf1d"
+uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
+version = "0.4.25"
+
 [[deps.JLLWrappers]]
 deps = ["Preferences"]
 git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1"
@@ -317,6 +329,12 @@ git-tree-sha1 = "dedbebe234e06e1ddad435f5c6f4b85cd8ce55f7"
 uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
 version = "2.2.2"
 
+[[deps.MacroTools]]
+deps = ["Markdown", "Random"]
+git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.9"
+
 [[deps.Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

Project.toml

@@ -10,6 +10,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Genie = "c43c736e-a2d1-11e8-161f-af95117fbd1e"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 Inflector = "6d011eab-0732-4556-8808-e463c76bf3b6"
+JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

app/resources/results/Results.jl

@@ -2,7 +2,7 @@ module Results
 
 using SearchLight
 using ..Main.UserApp.Surveys, Dates
-using DataFrames, CSV, JSON3, SQLite
+using DataFrames, CSV, JSON3, SQLite, JLD2
 
 export surveys, questions, responseids, results,
     register!, deregister!, save!, clear!
@@ -86,14 +86,15 @@ function response(survey::SurveyID, response::ResponseID; cache::Bool=true)
                            qid => Answer{anstype}(value, nothing)
                        end)
         metadata = SearchLight.query(
-            "SELECT started, completed, page \
+            "SELECT exip, started, completed, page \
              FROM responses WHERE survey=$survey AND id=$response")
+        exip = UInt32(metadata.exip[1])
         started = parse(DateTime, metadata.started[1])
         completed = if !ismissing(metadata.completed[1])
             parse(DateTime, metadata.completed[1]) end
         RESPONSES[survey][response] =
-            Response(survey, response, metadata.page[1], answers,
+            Response(survey, response, exip, metadata.page[1],
-                     started, completed)
+                     answers, started, completed)
     end
     RESPONSES[survey][response]
 end
@@ -111,6 +112,16 @@ function results(survey::SurveyID, resids::Vector{ResponseID};
     @info "" DataFrame(data)
     DataFrame(data) |> if format == :DataFrame
         identity
+    elseif format == :jld2
+        df -> (mktemp() do path, _
+                   df.id = resids
+                   jldsave(path; results = select(df, :id, :),
+                           responses = map(res) do r
+                               (; id, exip, started, completed, page) = r
+                           end |> DataFrame,
+                           questions = questions(survey; cache))
+                   read(path)
+               end)
     elseif format == :csv
         df -> sprint(CSV.write, df)
     elseif format == :tsv

app/resources/results/ResultsController.jl

@@ -28,6 +28,9 @@ function resultsfile(survey::SurveyID, format::AbstractString)
     elseif format == "db" || format == "sqlite"
         HTTP.Response(200, ["Content-Type" => "application/octet-stream"],
                       body = results(survey, format=:sqlite))
+    elseif format == "jld2"
+        HTTP.Response(200, ["Content-Type" => "application/x-hdf5"],
+                      body = results(survey, format=:jld2))
     else
         error("format $format not recognised")
     end
@@ -50,6 +53,9 @@ function resultsfile(survey::SurveyID, responseid::ResponseID, format::AbstractS
     elseif format == "db" || format == "sqlite"
         HTTP.Response(200, ["Content-Type" => "application/octet-stream"],
                       body = results(survey, [responseid], format=:sqlite))
+    elseif format == "jld2"
+        HTTP.Response(200, ["Content-Type" => "application/x-hdf5"],
+                      body = results(survey, [responseid], format=:jld2))
     else
         error("format $format not recognised")
     end

app/resources/results/views/survey.jl.html

@@ -15,12 +15,13 @@
     <hgroup>
         <h1>$(name)</h1>
         <span>
-            <strong>$(size(sresults, 1)) results also available as</strong>
+            <strong>$(size(sresults, 1)) results also available as </strong>
             <a href="$(id).txt" target="_blank">Text</a>,
             <a href="$(id).csv" target="_blank">CSV</a>,
             <a href="$(id).tsv" target="_blank">TSV</a>,
             <a href="$(id).json" target="_blank">JSON</a>,
-            <a href="$(id).db" target="_blank">SQLite DB</a>
+            <a href="$(id).db" target="_blank">SQLite DB</a>,
+            <a href="$(id).jld2" target="_blank">JLD2</a>
         </span>
     </hgroup>
 </header>

app/resources/surveys/SurveysController.jl

@@ -40,10 +40,13 @@ function new()
         @info "client ip: $ip"
         xor(reinterpret(UInt32, Genie.Encryption.encrypt(rand(UInt8, 4)) |> hex2bytes)...)
     end
-    r = Surveys.Response(SURVEY, vcat(responseids(SURVEY),
+    exip = encrypted_xord_ip()
-                                      Vector{Surveys.ResponseID}(keys(INPROGRESS) |> collect)))
+    r = Surveys.Response(SURVEY,
+                         vcat(responseids(SURVEY),
+                              Vector{Surveys.ResponseID}(keys(INPROGRESS) |> collect));
+                         exip)
     INPROGRESS[r.id] = r
-    register!(r, encrypted_xord_ip())
+    register!(r, exip)
     uid_str = string(r.id, base=UID_ENCBASE)
     Genie.Renderer.redirect(HTTP.URIs.URI(currenturl()).path * "?uid=$uid_str&page=1")
 end

lib/Surveys.jl

@@ -226,7 +226,8 @@ const ResponseID = UInt32
 mutable struct Response
     survey::SurveyID
     id::ResponseID
-    page::Integer
+    exip::UInt32
+    page::Int
     answers::Dict{Symbol, Answer}
     started::DateTime
     completed::Union{DateTime, Nothing}
@@ -242,18 +243,18 @@ Base.getindex(r::Response, id::Symbol) = r.answers[id]
 
 Answer(::Question{<:FormField{T}}) where {T} = Answer{T}(missing, nothing)
 
-Response(s::Survey, id::ResponseID=rand(ResponseID)) =
+Response(s::Survey, id::ResponseID=rand(ResponseID); exip::UInt32=zero(UInt32)) =
-    Response(s.id, id, 1,
+    Response(s.id, id, exip, 1,
              Dict(q.id => Answer(q) for q in
                       Iterators.flatten([s[i].questions for i in 1:length(s)])),
              now(), nothing)
 
-function Response(s::Survey, oldids::Vector{ResponseID})
+function Response(s::Survey, oldids::Vector{ResponseID}; exip::UInt32=zero(UInt32))
     newid = rand(ResponseID)
     while newid in oldids
         newid = rand(ResponseID)
     end
-    Response(s, newid)
+    Response(s, newid; exip)
 end
 
 interpret(::FormField{<:AbstractString}, value::AbstractString) = value

routes.jl

@@ -27,20 +27,20 @@ route("/results/:survey#([A-Za-z0-9]+)",
     ResultsController.resultsindex(surveyid)
 end
 
-route("/results/:surveyandformat#([A-Za-z0-9]+\\.[a-z]+)") do
+route("/results/:surveyandformat#([A-Za-z0-9]+\\.[a-z0-9]+)") do
     @info "" payload(:surveyandformat)
-    survey, format = match(r"([A-Za-z0-9]+)\.([a-z]+)", payload(:surveyandformat)).captures
+    survey, format = match(r"([A-Za-z0-9]+)\.([a-z0-9]+)", payload(:surveyandformat)).captures
     surveyid = tryparse(SurveysController.SurveyID, survey, base=10)
     ResultsController.resultsfile(surveyid, format)
 end
 
-route("/results/:survey#([A-Za-z0-9]+)/:format#([a-z]+)",
+route("/results/:survey#([A-Za-z0-9]+)/:format#([a-z0-9]+)",
       named = :surveyresult) do
     surveyid = tryparse(SurveysController.SurveyID, payload(:survey), base=10)
     ResultsController.resultsfile(surveyid, payload(:format))
 end
 
-route("/results/:survey#([A-Za-z0-9]+)/:responsefile#([A-Za-z0-9]+\\.[a-z]+)",
+route("/results/:survey#([A-Za-z0-9]+)/:responsefile#([A-Za-z0-9]+\\.[a-z0-9]+)",
       named = :result) do
     surveyid = tryparse(SurveysController.SurveyID, payload(:survey), base=10)
     response, format = split(payload(:responsefile), '.')