From 54b1aa5b8518a65c054fc4ef9f4c38410530a051 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Thu, 13 Feb 2020 18:51:45 -0500 Subject: [PATCH 001/651] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2bff5383e..d06243fed 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Writing analysis -This repository is for a writing analysis project. There isn't much to see here yet. +This repository is for a writing analysis project. There isn't much to see here yet. We're planning to look at writing processes in K-12 classrooms. Contact/maintainer: Piotr Mitros (pmitros@ets.org) -Licensing: Open source / free software. License TBD. \ No newline at end of file +Licensing: Open source / free software. License TBD. From 26cd26fe1d22852d0164dbf0617ecdb53ef59c73 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 5 Apr 2020 08:23:12 -0400 Subject: [PATCH 002/651] Experimental (incomplete) commit. --- configuration/tasks/writing.yaml | 1 + extension/background.js | 13 ++-- ux/deanne3.js | 113 +++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 9 deletions(-) create mode 100644 ux/deanne3.js diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing.yaml index c178b50ee..ae760c05a 100644 --- a/configuration/tasks/writing.yaml +++ b/configuration/tasks/writing.yaml @@ -49,3 +49,4 @@ - postgresql - nginx - certbot + - ejabberd \ No newline at end of file diff --git a/extension/background.js b/extension/background.js index 3c2f476c9..ad6051843 100644 --- a/extension/background.js +++ b/extension/background.js @@ -2,19 +2,14 @@ Background script. This works across all of Google Chrome. */ - -var event_queue = []; - -/* To avoid race conditions, we keep track of events we've successfully sent */ -var sent_events = new Set(); +var WRITINGJS_AJAX_SERVER = null; var webSocket = null; +var EXPERIMENTAL_WEBSOCKET = false; -//var WRITINGJS_AJAX_SERVER = "https://writing.hopto.org/webapi/"; -//var WRITINGJS_WSS_SERVER = "https://writing.hopto.org/webapi/"; -var WRITINGJS_AJAX_SERVER = null; -var EXPERIMENTAL_WEBSOCKET = false; +var event_queue = []; + /* FSM diff --git a/ux/deanne3.js b/ux/deanne3.js new file mode 100644 index 000000000..56dc12605 --- /dev/null +++ b/ux/deanne3.js @@ -0,0 +1,113 @@ +const LENGTH = 30; + +const width = 960; +const height = 500; +const margin = 5; +const padding = 5; +const adj = 30; + +function consecutive_array(n) { + /* + This creates an array of length n [0,1,2,3,4...n] + */ + return Array(n).fill().map((e,i)=>i+1); +}; + +function randn_bm() { + /* Approximately Gaussian distribution, mean 0.5 + From https://stackoverflow.com/questions/25582882/javascript-math-random-normal-distribution-gaussian-bell-curve */ + let u = 0, v = 0; + while(u === 0) u = Math.random(); //Converting [0,1) to (0,1) + while(v === 0) v = Math.random(); + let num = Math.sqrt( -2.0 * Math.log( u ) ) * Math.cos( 2.0 * Math.PI * v ); + num = num / 10.0 + 0.5; // Translate to 0 -> 1 + if (num > 1 || num < 0) return randn_bm(); // resample between 0 and 1 + return num; +} + + +function length_array(x) { + /* + Essay length + */ + return x.map((e,i)=> (e*randn_bm(e) + e)/2); +} + +function cursor_array(x) { + /* + Essay cursor position + */ + var length_array = x.map((e,i)=> (e*Math.random()/2 + e*randn_bm()/2)); + return length_array; +} + +function zip(a1, a2) { + return a1.map(function(e, i) { + return [e, a2[i]]; + }); +} + +function make_deanne_graph(div) { + var svg = d3.select(div).append("svg") + .attr("preserveAspectRatio", "xMinYMin meet") + .attr("viewBox", "-" + + adj + " -" + + adj + " " + + (width + adj *3) + " " + + (height + adj*3)) + .style("padding", padding) + .style("margin", margin) + .style("border", "1px solid lightgray") + .classed("svg-content", true); + + var x_edit = consecutive_array(LENGTH); + var y_length = length_array(x_edit); + var y_cursor = cursor_array(y_length); + + + const yScale = d3.scaleLinear().range([height, 0]).domain([0, LENGTH]) + const xScale = d3.scaleLinear().range([0, width]).domain([0, LENGTH]) + + + var xAxis = d3.axisBottom(xScale) + .ticks(4); // specify the number of ticks + var yAxis = d3.axisLeft(yScale) + .ticks(4); // specify the number of ticks + + svg.append('g') // create a element + .attr("transform", "translate(0, "+height+")") + .attr('class', 'x axis') // specify classes + .call(xAxis); // let the axis do its thing + + svg.append('g') // create a element + .attr('class', 'y axis') // specify classes + .call(yAxis); // let the axis do its thing + + var lines = d3.line(); + + var length_data = zip(x_edit.map(xScale), y_length.map(yScale)); + + var cursor_data = zip(x_edit.map(xScale), y_cursor.map(yScale)); + + var pathData = lines(length_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); + + pathData = lines(cursor_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); +} + +make_deanne_graph("#deanne") From f6dd4432b0261b40e5fdf12e10277a096da14a0f Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 5 Apr 2020 12:24:19 -0400 Subject: [PATCH 003/651] Considering refactor. Saving state --- ux/deanne.html | 16 +++ ux/deanne3.js | 22 ++-- ux/media/ETS_Logo.svg | 18 +++ ux/typing.html | 15 +++ ux/typing.js | 59 ++++++++++ ux/ux.css | 31 ++++++ ux/ux.html | 249 ++++++++++++++++++++++++++++++++++++++++++ ux/writing.js | 6 + 8 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 ux/deanne.html create mode 100644 ux/media/ETS_Logo.svg create mode 100644 ux/typing.html create mode 100644 ux/typing.js create mode 100644 ux/ux.css create mode 100644 ux/ux.html create mode 100644 ux/writing.js diff --git a/ux/deanne.html b/ux/deanne.html new file mode 100644 index 000000000..3ce2f4a1c --- /dev/null +++ b/ux/deanne.html @@ -0,0 +1,16 @@ + + + + + + + + +

Deanne

+
+
+ + + diff --git a/ux/deanne3.js b/ux/deanne3.js index 56dc12605..84dbad5f8 100644 --- a/ux/deanne3.js +++ b/ux/deanne3.js @@ -47,8 +47,10 @@ function zip(a1, a2) { }); } -function make_deanne_graph(div) { - var svg = d3.select(div).append("svg") +export const name = 'deanne3'; + +export function deanne_graph(div) { + var svg = div.append("svg") .attr("preserveAspectRatio", "xMinYMin meet") .attr("viewBox", "-" + adj + " -" @@ -78,7 +80,7 @@ function make_deanne_graph(div) { .attr("transform", "translate(0, "+height+")") .attr('class', 'x axis') // specify classes .call(xAxis); // let the axis do its thing - + svg.append('g') // create a element .attr('class', 'y axis') // specify classes .call(yAxis); // let the axis do its thing @@ -86,11 +88,11 @@ function make_deanne_graph(div) { var lines = d3.line(); var length_data = zip(x_edit.map(xScale), y_length.map(yScale)); - + var cursor_data = zip(x_edit.map(xScale), y_cursor.map(yScale)); - + var pathData = lines(length_data); - + svg.append('g') // create a element .attr('class', 'essay-length lines') .append('path') @@ -98,9 +100,9 @@ function make_deanne_graph(div) { .attr('fill', 'none') .attr('stroke', 'black') .attr('stroke-width','3'); - + pathData = lines(cursor_data); - + svg.append('g') // create a element .attr('class', 'essay-length lines') .append('path') @@ -108,6 +110,8 @@ function make_deanne_graph(div) { .attr('fill', 'none') .attr('stroke', 'black') .attr('stroke-width','3'); + + return svg; } -make_deanne_graph("#deanne") +d3.select("#deanne").call(deanne_graph).call(console.log); diff --git a/ux/media/ETS_Logo.svg b/ux/media/ETS_Logo.svg new file mode 100644 index 000000000..e6a0a1807 --- /dev/null +++ b/ux/media/ETS_Logo.svg @@ -0,0 +1,18 @@ + + + + + + + \ No newline at end of file diff --git a/ux/typing.html b/ux/typing.html new file mode 100644 index 000000000..1253eda43 --- /dev/null +++ b/ux/typing.html @@ -0,0 +1,15 @@ + + + + + + + +

Essay

+

+

+ + + diff --git a/ux/typing.js b/ux/typing.js new file mode 100644 index 000000000..4b0489a80 --- /dev/null +++ b/ux/typing.js @@ -0,0 +1,59 @@ +export const name = 'typing'; + +const SAMPLE_TEXT = "I like the goals of this petition and the bills, but as drafted, these bills just don't add up. We want to put our economy on hold. We definitely need a rent freeze. For that to work, we also need a mortgage freeze, not a mortgage forbearance. The difference is that in a mortgage forbearance, interest adds up and at the end, your principal is higher than when you started. In a mortgage freeze, the principal doesn't change -- you just literally push back all payments by a few months."; + +export function typing(div, ici=200, text=SAMPLE_TEXT) { + function randn_bm() { + /* Approximately Gaussian distribution, mean 0.5 + From https://stackoverflow.com/questions/25582882/javascript-math-random-normal-distribution-gaussian-bell-curve */ + let u = 0, v = 0; + while(u === 0) u = Math.random(); //Converting [0,1) to (0,1) + while(v === 0) v = Math.random(); + let num = Math.sqrt( -2.0 * Math.log( u ) ) * Math.cos( 2.0 * Math.PI * v ); + num = num / 10.0 + 0.5; // Translate to 0 -> 1 + if (num > 1 || num < 0) return randn_bm(); // resample between 0 and 1 + return num; + } + + function sample_ici(typing_delay=200) { + /* + Intercharacter interval -- how long between two keypresses + + We do an approximate Gaussian distribution around the + */ + return typing_delay * randn_bm() * 2; + } + + var start = 0; + var stop = 1; + const MAXIMUM_LENGTH = 300; + + function updateText() { + //document.getElementsByClassName("typing")[0].innerText=text.substr(start, stop-start); + div.text(text.substr(start, stop-start)); + stop = stop + 1; + + if(stop > text.length) { + stop = 1; + start = 0; + } + + start = Math.max(start, stop-MAXIMUM_LENGTH); + while((text[start] != ' ') && (start>0) && (startstop) { + start=stop; + } + + if(div.size() > 0) { + setTimeout(updateText, sample_ici(ici)); + }; + } + setTimeout(updateText, sample_ici(50)); +}; + +//typing(); + +d3.select(".typingdebug-typing").call(typing); + diff --git a/ux/ux.css b/ux/ux.css new file mode 100644 index 000000000..b8f336b76 --- /dev/null +++ b/ux/ux.css @@ -0,0 +1,31 @@ +/* Flip based on https://davidwalsh.name/css-flip */ +.wa-flip-container { + perspective: 1000px; +} + +.wa-flipper { + transition: 0.6s; + transform-style: preserve-3d; + + position: relative; +} + +.wa-front, .wa-back { + backface-visibility: hidden; + + position: absolute; + top: 20px; + left: 20px; +} + +/* front pane, placed above back */ +.wa-front { + z-index: 2; + /* for firefox 31 */ + transform: rotateY(0deg); +} + +/* back, initially hidden pane */ +.wa-back { + transform: rotateY(180deg); +} diff --git a/ux/ux.html b/ux/ux.html new file mode 100644 index 000000000..6e77ed1a8 --- /dev/null +++ b/ux/ux.html @@ -0,0 +1,249 @@ + + + + + + + + + + + + + + + + + + + + Writing Analysis + + + + + + + +
+
+
+ +
+ +
+ +
+
+

Irene

+

Subtitle

+

irene@kong.ma.us
+ 408 Wrangler Lane
+ 617-889-2292

+ + + + + + + + + +
+
+
+
+

Three

+

Subtitle

+
+
+
+
+
+
+

Four

+

Subtitle

+
+
+
+ +
+
+
+

One

+

Subtitle

+
+
+
+
+

Two

+

Subtitle

+
+
+
+
+

Three

+

Subtitle

+
+
+
+
+

Four

+

Subtitle

+
+
+
+ +
+
+
+

One

+

Subtitle

+
+
+
+
+

Two

+

Subtitle

+
+
+
+
+

Three

+

Subtitle

+
+
+
+
+

Four

+

Subtitle

+
+
+
+ +
+
+
+

One

+

Subtitle

+
+
+
+
+

Two

+

Subtitle

+
+
+
+
+

Three

+

Subtitle

+
+
+
+
+

Four

+

Subtitle

+
+
+
+ +
+
+
+

One

+

Subtitle

+
+
+
+
+

Two

+

Subtitle

+
+
+
+
+

Three

+

Subtitle

+
+
+
+
+

Four

+

Subtitle

+
+
+
+ + + +
+
+ + + + diff --git a/ux/writing.js b/ux/writing.js new file mode 100644 index 000000000..2b1900284 --- /dev/null +++ b/ux/writing.js @@ -0,0 +1,6 @@ +import { deanne_graph } from './deanne3.js' +import { typing } from './typing.js' + +d3.select(".typing").call(typing); + +console.log("Hello!"); From c1f8b271b155b5bca560e5a8fc6ed35b943ff348 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 6 Apr 2020 14:30:02 -0400 Subject: [PATCH 004/651] Progress --- ux/deanne.html | 3 +- ux/deanne3.js | 2 +- ux/summary_stats.html | 15 +++++++ ux/summary_stats.js | 80 +++++++++++++++++++++++++++++++++++++ ux/typing.js | 2 +- ux/ux.css | 8 ++++ ux/ux.html | 91 ++++++++++++++++++++++++++++++------------- ux/writing.js | 52 ++++++++++++++++++++++++- 8 files changed, 219 insertions(+), 34 deletions(-) create mode 100644 ux/summary_stats.html create mode 100644 ux/summary_stats.js diff --git a/ux/deanne.html b/ux/deanne.html index 3ce2f4a1c..f1b0b039e 100644 --- a/ux/deanne.html +++ b/ux/deanne.html @@ -7,8 +7,7 @@

Deanne

-
-
+
+ + + + +

Summary Stats

+
+ + + diff --git a/ux/summary_stats.js b/ux/summary_stats.js new file mode 100644 index 000000000..b49596915 --- /dev/null +++ b/ux/summary_stats.js @@ -0,0 +1,80 @@ +const LENGTH = 30; + +const width = 960; +const height = 500; +const margin = 5; +const padding = 5; +const adj = 30; + +export const name = 'summary_stats'; + +export function summary_stats(div) { + var svg = div.append("svg") + .attr("preserveAspectRatio", "xMinYMin meet") + .attr("viewBox", "-" + + adj + " -" + + adj + " " + + (width + adj *3) + " " + + (height + adj*3)) + .style("padding", padding) + .style("margin", margin) + .style("border", "1px solid lightgray") + .classed("svg-content", true); + + data = { + 'Active Time': 901, + 'Date Started': 5, + 'Characters Typed': 4065, + 'Text Complexity': 8, + 'Time Since Last Edit': 3, + 'Word Count': 678 + }; + + const yScale = d3.scaleLinear().range([height, 0]).domain([0, LENGTH]) + const xScale = d3.scaleLinear().range([0, width]).domain([0, LENGTH]) + + + var xAxis = d3.axisBottom(xScale) + .ticks(4); // specify the number of ticks + var yAxis = d3.axisLeft(yScale) + .ticks(4); // specify the number of ticks + + svg.append('g') // create a element + .attr("transform", "translate(0, "+height+")") + .attr('class', 'x axis') // specify classes + .call(xAxis); // let the axis do its thing + + svg.append('g') // create a element + .attr('class', 'y axis') // specify classes + .call(yAxis); // let the axis do its thing + + var lines = d3.line(); + + var length_data = zip(x_edit.map(xScale), y_length.map(yScale)); + + var cursor_data = zip(x_edit.map(xScale), y_cursor.map(yScale)); + + var pathData = lines(length_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); + + pathData = lines(cursor_data); + + svg.append('g') // create a element + .attr('class', 'essay-length lines') + .append('path') + .attr('d', pathData) + .attr('fill', 'none') + .attr('stroke', 'black') + .attr('stroke-width','3'); + + return svg; +} + +d3.select("#debug_testing_summary").call(summary_stats).call(console.log); diff --git a/ux/typing.js b/ux/typing.js index 4b0489a80..e9f30436f 100644 --- a/ux/typing.js +++ b/ux/typing.js @@ -26,7 +26,7 @@ export function typing(div, ici=200, text=SAMPLE_TEXT) { var start = 0; var stop = 1; - const MAXIMUM_LENGTH = 300; + const MAXIMUM_LENGTH = 250; function updateText() { //document.getElementsByClassName("typing")[0].innerText=text.substr(start, stop-start); diff --git a/ux/ux.css b/ux/ux.css index b8f336b76..29030635f 100644 --- a/ux/ux.css +++ b/ux/ux.css @@ -1,3 +1,11 @@ +.wa-row-tile { + min-height: 350px; +} + +.wa-col-tile { + min-height: 350px; +} + /* Flip based on https://davidwalsh.name/css-flip */ .wa-flip-container { perspective: 1000px; diff --git a/ux/ux.html b/ux/ux.html index 6e77ed1a8..c2eb5918c 100644 --- a/ux/ux.html +++ b/ux/ux.html @@ -19,9 +19,47 @@ Writing Analysis - + + + + + + + + + + + - +
- + +
+
diff --git a/ux/writing.js b/ux/writing.js index 2b1900284..3e9937826 100644 --- a/ux/writing.js +++ b/ux/writing.js @@ -1,6 +1,54 @@ import { deanne_graph } from './deanne3.js' import { typing } from './typing.js' -d3.select(".typing").call(typing); +var student_data = [[1,2,3,4],[5,6,7,8],[9,10,11,12],[13,14,15,16],[17,18,19]]; -console.log("Hello!"); +const tile_template = document.getElementById('template-tile').innerHTML + +function populate_tiles(tilesheet) { + var rows=tilesheet.selectAll("div.wa-row-tile") + .data(student_data) + .enter() + .append("div") + .attr("class", "tile is-ancestor wa-row-tile"); + + var cols=rows.selectAll("div.wa-col-tile") + .data(function(d) { return d; }) + .enter() + .append("div") + .attr("class", "tile is-parent wa-col-tile wa-flip-container is-3") + .html(function(d) { + return Mustache.render(tile_template, d); + /*{ + name: d.name, + body: document.getElementById('template-deanne-tile').innerHTML + });*/ + }) + .each(function(d) { + d3.select(this).select(".typing-text").call(typing, d.ici, d.essay); + }) + .each(function(d) { + d3.select(this).select(".deanne").call(deanne_graph); + }); +} + +function select_tab(tab) { + return function() { + d3.selectAll(".tilenav").classed("is-active", false); + d3.selectAll(".tilenav-"+tab).classed("is-active", true); + d3.selectAll(".wa-tilebody").classed("is-hidden", true); + d3.selectAll("."+tab).classed("is-hidden", false); + } +}; + +var tabs = ["typing", "deanne", "summary", "outline", "revision", "contact"]; +for(var i=0; i Date: Mon, 6 Apr 2020 16:57:04 -0400 Subject: [PATCH 005/651] Placeholders --- ux/ux.html | 32 +++++++++++++------------------- ux/writing.js | 2 +- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/ux/ux.html b/ux/ux.html index c2eb5918c..4d481c67f 100644 --- a/ux/ux.html +++ b/ux/ux.html @@ -25,18 +25,27 @@
+ + + - - - - - - - - - -

@@ -176,6 +175,7 @@

+ diff --git a/webapp/static/dashboard.js b/webapp/static/dashboard.js index da1882331..0e5685050 100644 --- a/webapp/static/dashboard.js +++ b/webapp/static/dashboard.js @@ -65,16 +65,20 @@ var ws = new WebSocket(`wss://${window.location.hostname}/wsapi/student-data/`) ws.onmessage = function (event) { console.log("Got data"); let data = JSON.parse(event.data); - student_data = data; //.student_data; - if(data.loggedin === false) { + // dispatch + if(data.logged_in === false) { d3.selectAll(".loading").classed("is-hidden", true); d3.selectAll(".auth-form").classed("is-hidden", false); d3.selectAll(".main").classed("is-hidden", true); - } else { + } else if (data.new_student_data) { + student_data = data.new_student_data; d3.selectAll(".loading").classed("is-hidden", true); d3.selectAll(".auth-form").classed("is-hidden", true); d3.selectAll(".main").classed("is-hidden", false); d3.select(".wa-tile-sheet").html(""); d3.select(".wa-tile-sheet").call(populate_tiles); + } else { + console.log(data); + console.log("Unrecognized JSON"); } }; diff --git a/webapp/student_data.py b/webapp/student_data.py index 7fe7e27cc..5cbbd580a 100644 --- a/webapp/student_data.py +++ b/webapp/student_data.py @@ -4,24 +4,42 @@ import synthetic_student_data +def authenticated(request): + ''' + Dummy function to tell if a request is logged in + ''' + return True -def static_student_data_handler(request): + +async def static_student_data_handler(request): ''' Populate static / mock-up dashboard with static fake data ''' - return aiohttp.web.json_response(json.load(open("static/student_data.js"))) + return aiohttp.web.json_response({ + "new_student_data": json.load(open("static/student_data.js")) + }) -def generated_student_data_handler(request): +async def generated_student_data_handler(request): ''' Populate static / mock-up dashboard with static fake data dynamically ''' - return aiohttp.web.json_response(synthetic_student_data.synthetic_data()) + return aiohttp.web.json_response({ + "new_student_data": synthetic_student_data.synthetic_data() + }) + async def ws_student_data_handler(request): print("Serving") ws = aiohttp.web.WebSocketResponse() await ws.prepare(request) - await ws.send_json(synthetic_student_data.synthetic_data()) + if authenticated(request): + await ws.send_json({ + "new_student_data": synthetic_student_data.synthetic_data() + }) + else: + await ws.send_json({ + "logged_in": False + }) student_data_handler = generated_student_data_handler From 5ae2769de4d2e71c6ff5945c0e4125ff18f9c8b4 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Tue, 9 Jun 2020 02:30:19 +0000 Subject: [PATCH 066/651] We can get data for many students. To confirm performance issues, etc. Also, cleanups for dynamic assessment. --- requirements.txt | 2 +- webapp/event_pipeline.py | 4 +- webapp/static/dashboard.js | 19 +++--- webapp/static/deane.js | 9 ++- webapp/static/typing.js | 9 +++ webapp/stream_analytics/helpers.py | 16 ++++- webapp/stream_writing.py | 3 +- webapp/student_data.py | 96 +++++++++++++++++++++++++++++- 8 files changed, 137 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e63c6acb..a147c19b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,4 @@ docopt pathvalidate names git+https://github.com/testlabauto/loremipsum.git@b7bd71a6651207ef88993045cd755f20747f2a1e#egg=loremipsmum -git+https://github.com/pmitros/tsvx.git@57efd1ab61114650326c5acdece2f850e86b2831#egg=tsvx +git+https://github.com/pmitros/tsvx.git@09bf7f33107f66413d929075a8b54c36ca581dae#egg=tsvx diff --git a/webapp/event_pipeline.py b/webapp/event_pipeline.py index ea032d3bb..b99d19119 100644 --- a/webapp/event_pipeline.py +++ b/webapp/event_pipeline.py @@ -239,8 +239,8 @@ async def incoming_websocket_handler(request): headers["test_framework_fake_identity"] = json_msg["user_id"] event_metadata['headers'].update(headers) - event_metadata['auth'] = await auth(headers) - print(event_metadata) + event_metadata['auth'] = await auth(headers) + print(event_metadata) event_handler = await handle_incoming_client_event(metadata=event_metadata) diff --git a/webapp/static/dashboard.js b/webapp/static/dashboard.js index 0e5685050..7d928d135 100644 --- a/webapp/static/dashboard.js +++ b/webapp/static/dashboard.js @@ -1,5 +1,5 @@ import { deane_graph } from './deane.js' -import { typing } from './typing.js' +import { student_text } from './text.js' import { summary_stats } from './summary_stats.js' import { outline } from './outline.js' @@ -27,10 +27,14 @@ function populate_tiles(tilesheet) { });*/ }) .each(function(d) { - d3.select(this).select(".typing-text").call(typing, d.ici, d.essay); + d3.select(this).select(".typing-text").call( + student_text, + d["stream_analytics.writing_analysis.reconstruct"].text); }) .each(function(d) { - d3.select(this).select(".deane").call(deane_graph); + d3.select(this).select(".deane").call( + deane_graph, + d["stream_analytics.writing_analysis.reconstruct"].edit_metadata); }) .each(function(d) { d3.select(this).select(".summary").call(summary_stats, d); @@ -54,23 +58,18 @@ for(var i=0; i Date: Sun, 19 Jul 2020 16:42:59 +0000 Subject: [PATCH 067/651] Basic Google auth --- webapp/aio_webapp_w.py | 42 ++++++- webapp/auth_handlers.py | 239 +++++++++++++++++++++++++++++++++++++++ webapp/static/index.html | 48 ++++++++ webapp/static/text.js | 36 ++++++ 4 files changed, 359 insertions(+), 6 deletions(-) create mode 100644 webapp/auth_handlers.py create mode 100644 webapp/static/index.html create mode 100644 webapp/static/text.js diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index 3ce9edbd9..a589f9ec4 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -11,12 +11,19 @@ import aiohttp import aiohttp_cors +import aiohttp_session +import aiohttp_session.cookie_storage + +import hashlib import pathvalidate import init # Odd import which makes sure we're set up import event_pipeline import student_data +import auth_handlers + +import settings routes = aiohttp.web.RouteTableDef() app = aiohttp.web.Application() @@ -32,11 +39,20 @@ async def request_logger_middleware(request, handler): app.on_response_prepare.append(request_logger_middleware) -def static_file_handler(basepath): +def static_file_handler(filename): ''' - Serve static files. + Serve a single static file + ''' + def handler(request): + return aiohttp.web.FileResponse(filename) + return handler + - This can be done directly by nginx on deployment. +def static_directory_handler(basepath): + ''' + Serve static files from a directory. + + This could be done directly by nginx on deployment. This is very minimal for now: No subdirectories, no gizmos, nothing fancy. I avoid fancy when we have user input and @@ -66,10 +82,10 @@ def handler(request): # Serve static files app.add_routes([ - aiohttp.web.get('/static/{filename}', static_file_handler("static")), - aiohttp.web.get('/static/media/{filename}', static_file_handler("media")), + aiohttp.web.get('/static/{filename}', static_directory_handler("static")), + aiohttp.web.get('/static/media/{filename}', static_directory_handler("media")), aiohttp.web.get('/static/media/avatar/{filename}', - static_file_handler("media/hubspot_persona_images/")), + static_directory_handler("media/hubspot_persona_images/")), ]) # Handle web sockets event requests, incoming and outgoing @@ -84,6 +100,11 @@ def handler(request): aiohttp.web.post('/webapi/event/', event_pipeline.ajax_event_request), ]) +app.add_routes([ + aiohttp.web.get('/', static_file_handler("static/index.html")), + aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social) +]) + cors = aiohttp_cors.setup(app, defaults={ "*": aiohttp_cors.ResourceOptions( allow_credentials=True, @@ -92,4 +113,13 @@ def handler(request): ) }) +def fernet_key(s): + t = hashlib.md5() + t.update(s.encode('utf-8')) + return t.hexdigest().encode('utf-8') + +aiohttp_session.setup(app, aiohttp_session.cookie_storage.EncryptedCookieStorage( + fernet_key(settings.settings['aio']['session_secret']), + max_age=settings.settings['aio']['session_max_age'])) + aiohttp.web.run_app(app, port=8888) diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py new file mode 100644 index 000000000..576ce190f --- /dev/null +++ b/webapp/auth_handlers.py @@ -0,0 +1,239 @@ +"""Authentication for Google. + +This was based on +[aiohttp-login](https://github.com/imbolc/aiohttp-login/), which at +the time worked with outdated Google APIs and require Jinja2. Oren +modernized this. Piotr integrated this into the system. + +Portions of this file, from aiohttp-login, are licensed as: + +Copyright (c) 2011 Imbolc. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Eventually, this should be broken out into its own module. +""" +import aiohttp +import aiohttp.web + +import logging +import os.path +import settings +#from aiohttp_jinja2 import template +#from aiohttp.abc import AbstractView +import aiohttp_session +#from functools import wraps +from yarl import URL + + + +async def social(request): + """Handles Google sign in. + + Provider is in `request.match_info['provider']` (currently, only Google) + """ + if request.match_info['provider'] != 'google': + raise aiohttp.web.HTTPMethodNotAllowed("We only handle Google logins") + + user = await _google(request) + + if 'user_id' in user: + # User ID returned in 'data', authorize user. + await _authorize_user(request, user) + url = user['back_to'] or "/" + return aiohttp.web.HTTPFound(url) + + return aiohttp.web.Response(text="Hello, world") + # No user ID returned from provider, Login failed. + #log.info(cfg['MSG_AUTH_FAILED']) + #return _redirect('auth_login') + + +# def user_to_request(handler): +# """ +# A handler function decorator that adds user to request if user logged in. +# :param handler: function to decorate. +# :return: decorated function +# """ +# @wraps(handler) +# async def decorator(*args): +# request = _get_request(args) +# request[cfg['REQUEST_USER_KEY']] = await _get_cur_user(request) +# return await handler(*args) +# return decorator + + +# def login_required(handler): +# """ +# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. +# :param handler: function to decorate. +# :return: decorated function +# """ +# @user_to_request +# @wraps(handler) +# async def decorator(*args): +# request = _get_request(args) +# if not request[cfg.REQUEST_USER_KEY]: +# return _redirect(_get_login_url(request)) +# return await handler(*args) +# return decorator + + +# @user_to_request +# @template('index.html') +# async def index(request): +# """Web app home page.""" +# return { +# 'auth': {'cfg': cfg}, +# 'cur_user': request['user'], +# 'url_for': _url_for, +# } + + +# @login_required +# @template('users.html') +# async def users(request): +# """Handles an example private page that requires logging in.""" +# return {} + + + + +# async def logout(request): +# """Handles sign out. This is generic - does not depend on which social ID is logged in +# (Google/Facebook/...).""" +# session = await aiohttp_session.get_session(request) +# session.pop(cfg["SESSION_USER_KEY"], None) +# return _redirect(cfg['LOGOUT_REDIRECT']) + + +async def _authorize_user(request, user): + """ + Logs a user in. + :param request: web request. + :param user_id: provider's user ID (e.g., Google ID). + """ + session = await aiohttp_session.get_session(request) + session["user_id"] = user + + +async def _google(request): + ''' + Handle Google login + ''' + if 'error' in request.query: + return {} + + common_params = { + 'client_id': settings.settings['google-oauth']['web']['client_id'], + 'redirect_uri': "https://writing.hopto.org/auth/login/google", + } + + # Step 1: redirect to get code + if 'code' not in request.query: + print("Here") + url = 'https://accounts.google.com/o/oauth2/auth' + params = common_params.copy() + params.update({ + 'response_type': 'code', + 'scope': ('https://www.googleapis.com/auth/userinfo.profile' + ' https://www.googleapis.com/auth/userinfo.email'), + }) + if 'back_to' in request.query: + params['state'] = request.query[back_to] + url = URL(url).with_query(params) + print(url) + raise aiohttp.web.HTTPFound(url) + + print("There") + # Step 2: get access token + url = 'https://accounts.google.com/o/oauth2/token' + params = common_params.copy() + params.update({ + 'client_secret': settings.settings['google-oauth']['web']['client_secret'], + 'code': request.query['code'], + 'grant_type': 'authorization_code', + }) + async with aiohttp.ClientSession(loop=request.app.loop) as client: + async with client.post(url, data=params) as resp: + data = await resp.json() + assert 'access_token' in data, data + + # get user profile + headers = {'Authorization': 'Bearer ' + data['access_token']} + # Old G+ URL that's no longer supported. + url = 'https://www.googleapis.com/oauth2/v1/userinfo' + async with client.get(url, headers=headers) as resp: + profile = await resp.json() + + return { + 'user_id': profile['id'], + 'email': profile['email'], + 'name': profile['given_name'], + 'family_name': profile['family_name'], + 'back_to': request.query.get('state'), + 'picture': profile['picture'], + } + + +# def _get_login_url(request): +# return _url_for('auth_login').with_query({ +# cfg['BACK_URL_QS_KEY']: request.path_qs}) + + +# async def _get_cur_user(request): +# user = await _get_cur_user_id(request) +# if user: +# # Load user object from database by the session user's user_id. This is disabled here, uncomment when we have +# # an underlying database. +# #user = await cfg.STORAGE.get_user({'id': user_id}) +# if not user: +# session = await aiohttp_session.get_session(request) +# del session['user'] +# return user + + +# async def _get_cur_user_id(request): +# session = await aiohttp_session.get_session(request) +# user = session.get(cfg['SESSION_USER_KEY']) +# while user: +# if not isinstance(user, dict): +# log.error('Wrong type of user_id in session') +# break + +# # Get a user ID from the user object. For now, we don't have a user database, so the session user is the same +# # as the "database-loaded" object user. Uncomment when we have a database. +# # user_id = cfg.STORAGE.user_id_from_string(user.user_id) +# # if not user_id: +# # break +# return user + +# if cfg['SESSION_USER_KEY'] in session: +# del session['user'] + + +# def _url_for(url_name, *args, **kwargs): +# if str(url_name).startswith(('/', 'http://', 'https://')): +# return url_name +# return cfg["APP"].router[url_name].url_for(*args, **kwargs) + + +# def _redirect(urlname, *args, **kwargs): +# return aiohttp.web.HTTPFound(_url_for(urlname, *args, **kwargs)) + + +# def _get_request(args): +# # Supports class based views see web.View +# if isinstance(args[0], AbstractView): +# return args[0].request +# return args[-1] diff --git a/webapp/static/index.html b/webapp/static/index.html new file mode 100644 index 000000000..d671f59c6 --- /dev/null +++ b/webapp/static/index.html @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + Writing Analysis + + + + +
+
+

Hello!

+ + + +
+
+
+
+

+ Writing Dashboard + by Piotr Mitros. Copyright + (c) 2020. Educational Testing + Service. The source + code will be released as free / open source software, + most likely + under the + AGPLv3 license. Our privacy policy. +

+
+
+ + diff --git a/webapp/static/text.js b/webapp/static/text.js new file mode 100644 index 000000000..557984405 --- /dev/null +++ b/webapp/static/text.js @@ -0,0 +1,36 @@ +export const name = 'student_text'; + +export function student_text(div, text) { + var start = 0; + var stop = 1; + const MAXIMUM_LENGTH = 250; + + div.text(text); +/* + .substr(start, stop-start)); + stop = stop + 1; + + if(stop > text.length) { + stop = 1; + start = 0; + } + + start = Math.max(start, stop-MAXIMUM_LENGTH); + while((text[start] != ' ') && (start>0) && (startstop) { + start=stop; + } + + if(div.size() > 0) { + setTimeout(updateText, sample_ici(ici)); + }; + } + setTimeout(updateText, sample_ici(50));*/ +} + +//typing(); + +d3.select(".textdebug-text").call(student_text); + From 4cd2d5f42537ddc29c622f940bc4b8b56fa61457 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 19 Jul 2020 12:44:57 -0400 Subject: [PATCH 068/651] Flags, minor changes --- configuration/local.yaml | 2 +- .../tasks/{writing.yaml => writing-apt.yaml} | 0 webapp/media/Flag_of_Poland.svg | 1 + webapp/media/Flag_of_the_United_States.svg | 26 +++++++++++++++++++ webapp/media/LICENSE.txt | 2 ++ 5 files changed, 30 insertions(+), 1 deletion(-) rename configuration/tasks/{writing.yaml => writing-apt.yaml} (100%) create mode 100644 webapp/media/Flag_of_Poland.svg create mode 100644 webapp/media/Flag_of_the_United_States.svg diff --git a/configuration/local.yaml b/configuration/local.yaml index d12102f87..3d502ca93 100644 --- a/configuration/local.yaml +++ b/configuration/local.yaml @@ -2,5 +2,5 @@ hosts: localhost connection: local tasks: - - include: tasks/writing.yaml + - include: tasks/writing-apt.yaml diff --git a/configuration/tasks/writing.yaml b/configuration/tasks/writing-apt.yaml similarity index 100% rename from configuration/tasks/writing.yaml rename to configuration/tasks/writing-apt.yaml diff --git a/webapp/media/Flag_of_Poland.svg b/webapp/media/Flag_of_Poland.svg new file mode 100644 index 000000000..b08d02519 --- /dev/null +++ b/webapp/media/Flag_of_Poland.svg @@ -0,0 +1 @@ + diff --git a/webapp/media/Flag_of_the_United_States.svg b/webapp/media/Flag_of_the_United_States.svg new file mode 100644 index 000000000..a11cf5f94 --- /dev/null +++ b/webapp/media/Flag_of_the_United_States.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/webapp/media/LICENSE.txt b/webapp/media/LICENSE.txt index 6b40ec588..c3d2d6697 100644 --- a/webapp/media/LICENSE.txt +++ b/webapp/media/LICENSE.txt @@ -7,3 +7,5 @@ https://www.ets.org/legal/trademarks/owned It is not distributed under the same license as the rest of this system. + +Flags of Poland and the US are SVGs from Wikipedia, and in the public domain From 24cfab4d8935d97f93eca53826ca959a84be696f Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 7 Aug 2020 01:58:56 +0000 Subject: [PATCH 069/651] auth works --- webapp/aio_webapp_w.py | 11 ++- webapp/auth_handlers.py | 175 ++++++++++++++-------------------------- 2 files changed, 71 insertions(+), 115 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index a589f9ec4..c392f14c4 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -43,7 +43,10 @@ def static_file_handler(filename): ''' Serve a single static file ''' - def handler(request): + @auth_handlers.user_to_request + async def handler(request): + foo = await aiohttp_session.get_session(request) + print("Foo:", foo.get("user", {})) return aiohttp.web.FileResponse(filename) return handler @@ -75,6 +78,7 @@ def handler(request): # Student data API +# This serves up data (currently usually dummy data) for the dashboard app.add_routes([ aiohttp.web.get('/webapi/student-data/', student_data.student_data_handler), aiohttp.web.get('/wsapi/student-data/', student_data.ws_student_data_handler) @@ -100,9 +104,12 @@ def handler(request): aiohttp.web.post('/webapi/event/', event_pipeline.ajax_event_request), ]) +# Generic web-appy things app.add_routes([ aiohttp.web.get('/', static_file_handler("static/index.html")), - aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social) + aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social), + aiohttp.web.get('/auth/logout', handler=auth_handlers.logout), + aiohttp.web.get('/auth/userinfo', handler=auth_handlers.user_info) ]) cors = aiohttp_cors.setup(app, defaults={ diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index 576ce190f..fed270f50 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -32,11 +32,10 @@ #from aiohttp_jinja2 import template #from aiohttp.abc import AbstractView import aiohttp_session -#from functools import wraps +from functools import wraps from yarl import URL - async def social(request): """Handles Google sign in. @@ -46,6 +45,7 @@ async def social(request): raise aiohttp.web.HTTPMethodNotAllowed("We only handle Google logins") user = await _google(request) + print(user) if 'user_id' in user: # User ID returned in 'data', authorize user. @@ -53,68 +53,8 @@ async def social(request): url = user['back_to'] or "/" return aiohttp.web.HTTPFound(url) - return aiohttp.web.Response(text="Hello, world") - # No user ID returned from provider, Login failed. - #log.info(cfg['MSG_AUTH_FAILED']) - #return _redirect('auth_login') - - -# def user_to_request(handler): -# """ -# A handler function decorator that adds user to request if user logged in. -# :param handler: function to decorate. -# :return: decorated function -# """ -# @wraps(handler) -# async def decorator(*args): -# request = _get_request(args) -# request[cfg['REQUEST_USER_KEY']] = await _get_cur_user(request) -# return await handler(*args) -# return decorator - - -# def login_required(handler): -# """ -# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. -# :param handler: function to decorate. -# :return: decorated function -# """ -# @user_to_request -# @wraps(handler) -# async def decorator(*args): -# request = _get_request(args) -# if not request[cfg.REQUEST_USER_KEY]: -# return _redirect(_get_login_url(request)) -# return await handler(*args) -# return decorator - - -# @user_to_request -# @template('index.html') -# async def index(request): -# """Web app home page.""" -# return { -# 'auth': {'cfg': cfg}, -# 'cur_user': request['user'], -# 'url_for': _url_for, -# } - - -# @login_required -# @template('users.html') -# async def users(request): -# """Handles an example private page that requires logging in.""" -# return {} - - - - -# async def logout(request): -# """Handles sign out. This is generic - does not depend on which social ID is logged in -# (Google/Facebook/...).""" -# session = await aiohttp_session.get_session(request) -# session.pop(cfg["SESSION_USER_KEY"], None) -# return _redirect(cfg['LOGOUT_REDIRECT']) + ## Login failed. TODO: Make a proper login failed page. + return aiohttp.web.HTTPFound("/") async def _authorize_user(request, user): @@ -124,7 +64,36 @@ async def _authorize_user(request, user): :param user_id: provider's user ID (e.g., Google ID). """ session = await aiohttp_session.get_session(request) - session["user_id"] = user + session["user"] = user + + +async def logout(request): + """Handles sign out. This is generic - does not depend on which social ID is logged in + (Google/Facebook/...).""" + session = await aiohttp_session.get_session(request) + session.pop("user", None) + return aiohttp.web.HTTPFound("/") ## TODO: Make a proper logout page + + + +def user_to_request(handler): + """ + A handler function decorator that adds user to request if user logged in. + :param handler: function to decorate. + :return: decorated function + """ + @wraps(handler) + async def decorator(*args): + request = args[0] + session = await aiohttp_session.get_session(request) + request['user'] = session.get('user', None) + return await handler(*args) + return decorator + + +@user_to_request +async def user_info(request): + return aiohttp.web.json_response(request['user']) async def _google(request): @@ -141,7 +110,6 @@ async def _google(request): # Step 1: redirect to get code if 'code' not in request.query: - print("Here") url = 'https://accounts.google.com/o/oauth2/auth' params = common_params.copy() params.update({ @@ -152,10 +120,8 @@ async def _google(request): if 'back_to' in request.query: params['state'] = request.query[back_to] url = URL(url).with_query(params) - print(url) raise aiohttp.web.HTTPFound(url) - print("There") # Step 2: get access token url = 'https://accounts.google.com/o/oauth2/token' params = common_params.copy() @@ -186,54 +152,37 @@ async def _google(request): } -# def _get_login_url(request): -# return _url_for('auth_login').with_query({ -# cfg['BACK_URL_QS_KEY']: request.path_qs}) - - -# async def _get_cur_user(request): -# user = await _get_cur_user_id(request) -# if user: -# # Load user object from database by the session user's user_id. This is disabled here, uncomment when we have -# # an underlying database. -# #user = await cfg.STORAGE.get_user({'id': user_id}) -# if not user: -# session = await aiohttp_session.get_session(request) -# del session['user'] -# return user - - -# async def _get_cur_user_id(request): -# session = await aiohttp_session.get_session(request) -# user = session.get(cfg['SESSION_USER_KEY']) -# while user: -# if not isinstance(user, dict): -# log.error('Wrong type of user_id in session') -# break - -# # Get a user ID from the user object. For now, we don't have a user database, so the session user is the same -# # as the "database-loaded" object user. Uncomment when we have a database. -# # user_id = cfg.STORAGE.user_id_from_string(user.user_id) -# # if not user_id: -# # break -# return user - -# if cfg['SESSION_USER_KEY'] in session: -# del session['user'] +# def login_required(handler): +# """ +# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. +# :param handler: function to decorate. +# :return: decorated function +# """ +# @user_to_request +# @wraps(handler) +# async def decorator(*args): +# request = _get_request(args) +# if not request[cfg.REQUEST_USER_KEY]: +# return _redirect(_get_login_url(request)) +# return await handler(*args) +# return decorator -# def _url_for(url_name, *args, **kwargs): -# if str(url_name).startswith(('/', 'http://', 'https://')): -# return url_name -# return cfg["APP"].router[url_name].url_for(*args, **kwargs) +# @user_to_request +# @template('index.html') +# async def index(request): +# """Web app home page.""" +# return { +# 'auth': {'cfg': cfg}, +# 'cur_user': request['user'], +# 'url_for': _url_for, +# } -# def _redirect(urlname, *args, **kwargs): -# return aiohttp.web.HTTPFound(_url_for(urlname, *args, **kwargs)) +# @login_required +# @template('users.html') +# async def users(request): +# """Handles an example private page that requires logging in.""" +# return {} -# def _get_request(args): -# # Supports class based views see web.View -# if isinstance(args[0], AbstractView): -# return args[0].request -# return args[-1] From 943b59830f8b5bc4e809472b4b0be20b2dd9cadb Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 7 Aug 2020 02:12:07 +0000 Subject: [PATCH 070/651] Moved auth into middleware --- webapp/aio_webapp_w.py | 2 ++ webapp/auth_handlers.py | 24 ++++++++++-------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index c392f14c4..68fb5c5b3 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -129,4 +129,6 @@ def fernet_key(s): fernet_key(settings.settings['aio']['session_secret']), max_age=settings.settings['aio']['session_max_age'])) +app.middlewares.append(auth_handlers.auth_middleware) + aiohttp.web.run_app(app, port=8888) diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index fed270f50..4f7589988 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -75,23 +75,19 @@ async def logout(request): return aiohttp.web.HTTPFound("/") ## TODO: Make a proper logout page +@aiohttp.web.middleware +async def auth_middleware(request, handler): + ''' + Move user into the request -def user_to_request(handler): - """ - A handler function decorator that adds user to request if user logged in. - :param handler: function to decorate. - :return: decorated function - """ - @wraps(handler) - async def decorator(*args): - request = args[0] - session = await aiohttp_session.get_session(request) - request['user'] = session.get('user', None) - return await handler(*args) - return decorator + Save user into a cookie + ''' + session = await aiohttp_session.get_session(request) + request['user'] = session.get('user', None) + resp = await handler(request) + return resp -@user_to_request async def user_info(request): return aiohttp.web.json_response(request['user']) From 094a9168258a02853cda9a69a203ba93f22b357b Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Fri, 7 Aug 2020 19:44:35 +0000 Subject: [PATCH 071/651] Auth middleware, no decorator --- webapp/aio_webapp_w.py | 3 --- webapp/auth_handlers.py | 39 ++++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index 68fb5c5b3..f2aa47ae1 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -43,10 +43,7 @@ def static_file_handler(filename): ''' Serve a single static file ''' - @auth_handlers.user_to_request async def handler(request): - foo = await aiohttp_session.get_session(request) - print("Foo:", foo.get("user", {})) return aiohttp.web.FileResponse(filename) return handler diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index 4f7589988..c243e193e 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -35,6 +35,9 @@ from functools import wraps from yarl import URL +import json +import base64 + async def social(request): """Handles Google sign in. @@ -85,6 +88,23 @@ async def auth_middleware(request, handler): session = await aiohttp_session.get_session(request) request['user'] = session.get('user', None) resp = await handler(request) + if request['user'] is None: + userinfo = None + else: + userinfo = { + "name": request['user']['name'], + "picture": request['user']['picture'] + } + # This is a dumb way to sanitize data and pass to JS. + # + # Cookies tend to get encoded and decoded in ad-hoc strings a lot, often + # in non-compliant ways (to see why, try to find the spec for cookies!) + # + # This avoids bugs (and, should the issue come up, injections) + # + # This should really be abstracted away into a library which passes state + # back-and-forth. + resp.set_cookie("userinfo", base64.b64encode(json.dumps(userinfo).encode('utf-8')).decode('utf-8')) return resp @@ -163,22 +183,3 @@ async def _google(request): # return await handler(*args) # return decorator - -# @user_to_request -# @template('index.html') -# async def index(request): -# """Web app home page.""" -# return { -# 'auth': {'cfg': cfg}, -# 'cur_user': request['user'], -# 'url_for': _url_for, -# } - - -# @login_required -# @template('users.html') -# async def users(request): -# """Handles an example private page that requires logging in.""" -# return {} - - From 11ea0c81f398a21a44e5720367e53f263bb25e41 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Tue, 18 Aug 2020 17:36:57 +0000 Subject: [PATCH 072/651] Teachers template, better startup logic --- webapp/init.py | 52 +++++++++++++++++++++++ webapp/static_data/teachers.yaml.template | 19 +++++++++ 2 files changed, 71 insertions(+) create mode 100644 webapp/static_data/teachers.yaml.template diff --git a/webapp/init.py b/webapp/init.py index 87c1bdf30..c38d6260e 100644 --- a/webapp/init.py +++ b/webapp/init.py @@ -1,8 +1,23 @@ +''' +This file mostly confirms we have prerequisites for the system to work. + +We create a logs directory, grab 3rd party libraries, etc. +''' + +import hashlib import os +import shutil import sys if not os.path.exists("logs"): os.mkdir("logs") + print("Made logs directory") + + +if not os.path.exists("static_data/teachers.yaml"): + shutil.copyfile("static_data/teachers.yaml.template", "static_data/teachers.yaml") + print("Created a blank teachers file: static_data/teachers.yaml\n" + "Populate it with teacher accounts.") if not os.path.exists("../creds.yaml"): print(""" @@ -13,3 +28,40 @@ Fill in the missing fields. """) sys.exit(-1) + +if not os.path.exists("static/3rd_party"): + os.mkdir("static/3rd_party") + for name, url, sha in [ + ("require.js", "https://requirejs.org/docs/release/2.3.6/comments/require.js", "d1e7687c1b2990966131bc25a761f03d6de83115512c9ce85d72e4b9819fb8733463fa0d93ca31e2e42ebee6e425d811e3420a788a0fc95f745aa349e3b01901"), + ("bulma.min.css", "https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.0/css/bulma.min.css", "ec7342883fdb6fbd4db80d7b44938951c3903d2132fc3e4bf7363c6e6dc5295a478c930856177ac6257c32e1d1e10a4132c6c51d194b3174dc670ab8d116b362"), + ("fontawesome.js", "https://use.fontawesome.com/releases/v5.3.1/js/all.js -O fontawesome.js", "83e7b36f1545d5abe63bea9cd3505596998aea272dd05dee624b9a2c72f9662618d4bff6e51fafa25d41cb59bd97f3ebd72fd94ebd09a52c17c4c23fdca3962b"), + ("showdown.js", "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js", "4fe14f17c2a1d0275d44e06d7e68d2b177779196c6d0c562d082eb5435eec4e710a625be524767aef3d9a1f6a5b88f912ddd71821f4a9df12ff7dd66d6fbb3c9"), + ("mustache.min.js", "http://cdnjs.cloudflare.com/ajax/libs/mustache.js/3.1.0/mustache.min.js", "e7c446dc9ac2da9396cf401774efd9bd063d25920343eaed7bee9ad878840e846d48204d62755aede6f51ae6f169dcc9455f45c1b86ba1b42980ccf8f241af25"), + ("d3.v5.min.js", "https://d3js.org/d3.v5.min.js", "466fe57816d719048885357cccc91a082d8e5d3796f227f88a988bf36a5c2ceb7a4d25842f5f3c327a0151d682e648cd9623bfdcc7a18a70ac05cfd0ec434463"), + ]: + filename = "static/3rd_party/{name}".format(name=name) + os.system("wget {url} -O {filename} 2> /dev/null".format( + url=url, + filename=filename + )) + shahash = hashlib.sha3_512(open(filename, "rb").read()).hexdigest() + print("Downloaded {name}".format(name=name)) + if shahash == sha: + print("File integrity confirmed!") + else: + print("Incorrect SHA hash. Something odd is going on. DO NOT IGNORE THIS ERROR/WARNING") + print() + print("Expected SHA: " + sha) + print("Actual SHA: " + shahash) + print() + print("We download 3rd party libraries from the Internet. This error means that ones of") + print("these files changed. This may indicate a man-in-the-middle attack, that a CDN has") + print("been compromised, or more prosaically, that one of the files had something like") + print("a security fix backported. In either way, VERIFY what happened before moving on.") + print("If unsure, please consult with a security expert.") + print() + print("This error should never happen unless someone is under attack (or there is a") + print("serious bug).") + os.exit(-1) + + print() diff --git a/webapp/static_data/teachers.yaml.template b/webapp/static_data/teachers.yaml.template new file mode 100644 index 000000000..15fa3dd3b --- /dev/null +++ b/webapp/static_data/teachers.yaml.template @@ -0,0 +1,19 @@ +# This is a template YAML file of authorized teachers. +# +# Each line contains one teacher: +# +# { +# "vsweeny@polk.apsd.us": {"google_id": "1234567890"}, +# "mfornz@polk.apsd.us": {"google_id": "1234567891"}, +# "jlee@polk.apsd.us": {"google_id": "1234567891"} +# } +# +# This should be the official Google Classroom email address, and the +# Google ID of the teacher. +# +# The first time this application is run, this template will be used +# to create a teachers.yaml file. You should not edit this one, but +# that one. + +{ +} From f9900b8634e2b18a3e6733a9c663f9ca6fbfd6ce Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Tue, 18 Aug 2020 17:37:22 +0000 Subject: [PATCH 073/651] Ignore commiting data file --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 988ebdfcc..c0904debe 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ .\#* *__pycache__* webapp/logs -creds.yaml \ No newline at end of file +webapp/static_data/teachers.yaml +creds.yaml From 1f66a67780f1bde14fa0bad3fd4fcc001a97f9e3 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sat, 22 Aug 2020 12:05:06 +0000 Subject: [PATCH 074/651] Support for Google Classroom, class rosters, front-end rendering, etc. Also, adding scripts to generate test data. Also, cleaning up initialization scripts --- webapp/aio_webapp_w.py | 19 ++- webapp/auth_handlers.py | 88 +++++++++--- webapp/init.py | 4 +- webapp/rosters.py | 87 +++++++++++ webapp/static/index.html | 2 +- webapp/static/modules/course.html | 45 ++++++ webapp/static/modules/courses.html | 21 +++ webapp/static/modules/login.html | 3 + webapp/static/modules/navbar_loggedin.html | 17 +++ webapp/static/webapp.html | 55 +++++++ webapp/static/webapp.js | 136 ++++++++++++++++++ webapp/static_data/README.md | 31 ++++ .../static_data/make_dummy_test_user_tsv.py | 26 ++++ .../make_google_classroom_test_courses.py | 78 ++++++++++ 14 files changed, 587 insertions(+), 25 deletions(-) create mode 100644 webapp/rosters.py create mode 100644 webapp/static/modules/course.html create mode 100644 webapp/static/modules/courses.html create mode 100644 webapp/static/modules/login.html create mode 100644 webapp/static/modules/navbar_loggedin.html create mode 100644 webapp/static/webapp.html create mode 100644 webapp/static/webapp.js create mode 100644 webapp/static_data/README.md create mode 100644 webapp/static_data/make_dummy_test_user_tsv.py create mode 100644 webapp/static_data/make_google_classroom_test_courses.py diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index f2aa47ae1..adafc6186 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -22,6 +22,7 @@ import event_pipeline import student_data import auth_handlers +import rosters import settings @@ -48,6 +49,17 @@ async def handler(request): return handler +async def index(request): + print(request['user']) + print(type(request['user'])) + if request['user'] is None: + print("Index") + return aiohttp.web.FileResponse("static/index.html") + else: + print("Course list") + return aiohttp.web.FileResponse("static/courselist.html") + + def static_directory_handler(basepath): ''' Serve static files from a directory. @@ -84,6 +96,8 @@ def handler(request): # Serve static files app.add_routes([ aiohttp.web.get('/static/{filename}', static_directory_handler("static")), + aiohttp.web.get('/static/modules/{filename}', static_directory_handler("static/modules")), + aiohttp.web.get('/static/3rd_party/{filename}', static_directory_handler("static/3rd_party")), aiohttp.web.get('/static/media/{filename}', static_directory_handler("media")), aiohttp.web.get('/static/media/avatar/{filename}', static_directory_handler("media/hubspot_persona_images/")), @@ -99,11 +113,14 @@ def handler(request): app.add_routes([ aiohttp.web.get('/webapi/event/', event_pipeline.ajax_event_request), aiohttp.web.post('/webapi/event/', event_pipeline.ajax_event_request), + aiohttp.web.get('/webapi/courselist/', rosters.courselist_api), + aiohttp.web.get('/webapi/courseroster/{course_id}', rosters.courseroster_api), ]) # Generic web-appy things app.add_routes([ - aiohttp.web.get('/', static_file_handler("static/index.html")), +# aiohttp.web.get('/', static_file_handler("static/index.html")), + aiohttp.web.get('/', index), aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social), aiohttp.web.get('/auth/logout', handler=auth_handlers.logout), aiohttp.web.get('/auth/userinfo', handler=auth_handlers.user_info) diff --git a/webapp/auth_handlers.py b/webapp/auth_handlers.py index c243e193e..cb375ebde 100644 --- a/webapp/auth_handlers.py +++ b/webapp/auth_handlers.py @@ -36,8 +36,29 @@ from yarl import URL import json +import yaml import base64 +import rosters + + +async def verify_teacher_account(email, google_id): + ''' + Confirm the teacher is registered with the system. Eventually, we will want + 3 versions of this: + * Always true (open system) + * Text file backed (pilots, small deploys) + * Database-backed (large-scale deploys) + + For now, we have the file-backed version + ''' + users = yaml.safe_load(open("static_data/teachers.yaml")) + if email not in users: + return False + if users[email]["google_id"] != google_id: + return False + return True + async def social(request): """Handles Google sign in. @@ -52,9 +73,12 @@ async def social(request): if 'user_id' in user: # User ID returned in 'data', authorize user. - await _authorize_user(request, user) - url = user['back_to'] or "/" - return aiohttp.web.HTTPFound(url) + authorized = await _authorize_user(request, user) + if authorized: + url = user['back_to'] or "/" + return aiohttp.web.HTTPFound(url) + else: + url = "/static/unauth.html" ## Login failed. TODO: Make a proper login failed page. return aiohttp.web.HTTPFound("/") @@ -68,13 +92,17 @@ async def _authorize_user(request, user): """ session = await aiohttp_session.get_session(request) session["user"] = user + return verify_teacher_account(user['user_id'], user['email']) async def logout(request): """Handles sign out. This is generic - does not depend on which social ID is logged in - (Google/Facebook/...).""" + (Google/Facebook/...). + """ session = await aiohttp_session.get_session(request) session.pop("user", None) + session.pop("auth_headers", None) + print(session) return aiohttp.web.HTTPFound("/") ## TODO: Make a proper logout page @@ -87,6 +115,7 @@ async def auth_middleware(request, handler): ''' session = await aiohttp_session.get_session(request) request['user'] = session.get('user', None) + request['auth_headers'] = session.get('auth_headers', None) resp = await handler(request) if request['user'] is None: userinfo = None @@ -95,15 +124,16 @@ async def auth_middleware(request, handler): "name": request['user']['name'], "picture": request['user']['picture'] } - # This is a dumb way to sanitize data and pass to JS. + # This is a dumb way to sanitize data and pass it to the front-end. # # Cookies tend to get encoded and decoded in ad-hoc strings a lot, often # in non-compliant ways (to see why, try to find the spec for cookies!) # - # This avoids bugs (and, should the issue come up, injections) + # This avoids bugs (and, should the issue come up, security issues + # like injections) # # This should really be abstracted away into a library which passes state - # back-and-forth. + # back-and-forth, but for now, this works. resp.set_cookie("userinfo", base64.b64encode(json.dumps(userinfo).encode('utf-8')).decode('utf-8')) return resp @@ -131,7 +161,9 @@ async def _google(request): params.update({ 'response_type': 'code', 'scope': ('https://www.googleapis.com/auth/userinfo.profile' - ' https://www.googleapis.com/auth/userinfo.email'), + ' https://www.googleapis.com/auth/userinfo.email' + ' https://www.googleapis.com/auth/classroom.courses.readonly' + ' https://www.googleapis.com/auth/classroom.rosters.readonly'), }) if 'back_to' in request.query: params['state'] = request.query[back_to] @@ -153,10 +185,20 @@ async def _google(request): # get user profile headers = {'Authorization': 'Bearer ' + data['access_token']} + session = await aiohttp_session.get_session(request) + session["auth_headers"] = headers + request["auth_headers"] = headers + # Old G+ URL that's no longer supported. url = 'https://www.googleapis.com/oauth2/v1/userinfo' async with client.get(url, headers=headers) as resp: profile = await resp.json() + print(profile) + for course in (await rosters.courselist(request)): + print(json.dumps(course, indent=3)) + print(await rosters.courseroster(request, course['id'])) + #async with client.get('"), headers=headers) as resp: + # print(await resp.text()) return { 'user_id': profile['id'], @@ -168,18 +210,20 @@ async def _google(request): } -# def login_required(handler): -# """ -# A handler function decorator that enforces that the user is logged in. If not, redirects to the login page. -# :param handler: function to decorate. -# :return: decorated function -# """ -# @user_to_request -# @wraps(handler) -# async def decorator(*args): -# request = _get_request(args) -# if not request[cfg.REQUEST_USER_KEY]: -# return _redirect(_get_login_url(request)) -# return await handler(*args) -# return decorator +def html_login_required(handler): + """ + A handler function decorator that enforces that the user is logged + in. If not, redirects to the login page. + + :param handler: function to decorate. + :return: decorated function + + """ + @wraps(handler) + async def decorator(*args): + user = args[0]["user"] + if user is None: + return aiohttp.web.HTTPFound("/") + return handler(*args) + return decorator diff --git a/webapp/init.py b/webapp/init.py index c38d6260e..6f37278c5 100644 --- a/webapp/init.py +++ b/webapp/init.py @@ -33,9 +33,11 @@ os.mkdir("static/3rd_party") for name, url, sha in [ ("require.js", "https://requirejs.org/docs/release/2.3.6/comments/require.js", "d1e7687c1b2990966131bc25a761f03d6de83115512c9ce85d72e4b9819fb8733463fa0d93ca31e2e42ebee6e425d811e3420a788a0fc95f745aa349e3b01901"), + ("text.js", "https://raw.githubusercontent.com/requirejs/text/3f9d4c19b3a1a3c6f35650c5788cbea1db93197a/text.js", "fb8974f1633f261f77220329c7070ff214241ebd33a1434f2738572608efc8eb6699961734285e9500bbbd60990794883981fb113319503208822e6706bca0b8"), ("bulma.min.css", "https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.0/css/bulma.min.css", "ec7342883fdb6fbd4db80d7b44938951c3903d2132fc3e4bf7363c6e6dc5295a478c930856177ac6257c32e1d1e10a4132c6c51d194b3174dc670ab8d116b362"), ("fontawesome.js", "https://use.fontawesome.com/releases/v5.3.1/js/all.js -O fontawesome.js", "83e7b36f1545d5abe63bea9cd3505596998aea272dd05dee624b9a2c72f9662618d4bff6e51fafa25d41cb59bd97f3ebd72fd94ebd09a52c17c4c23fdca3962b"), ("showdown.js", "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js", "4fe14f17c2a1d0275d44e06d7e68d2b177779196c6d0c562d082eb5435eec4e710a625be524767aef3d9a1f6a5b88f912ddd71821f4a9df12ff7dd66d6fbb3c9"), + ("showdown.js.map", "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js.map", "74690aa3cea07fd075942ba9e98cf7297752994b93930acb3a1baa2d3042a62b5523d3da83177f63e6c02fe2a09c8414af9e1774dad892a303e15a86dbeb29ba"), ("mustache.min.js", "http://cdnjs.cloudflare.com/ajax/libs/mustache.js/3.1.0/mustache.min.js", "e7c446dc9ac2da9396cf401774efd9bd063d25920343eaed7bee9ad878840e846d48204d62755aede6f51ae6f169dcc9455f45c1b86ba1b42980ccf8f241af25"), ("d3.v5.min.js", "https://d3js.org/d3.v5.min.js", "466fe57816d719048885357cccc91a082d8e5d3796f227f88a988bf36a5c2ceb7a4d25842f5f3c327a0151d682e648cd9623bfdcc7a18a70ac05cfd0ec434463"), ]: @@ -62,6 +64,6 @@ print() print("This error should never happen unless someone is under attack (or there is a") print("serious bug).") - os.exit(-1) + sys.exit(-1) print() diff --git a/webapp/rosters.py b/webapp/rosters.py new file mode 100644 index 000000000..0633b825b --- /dev/null +++ b/webapp/rosters.py @@ -0,0 +1,87 @@ +import aiohttp +import aiohttp.web + +import settings + +COURSE_URL = 'https://classroom.googleapis.com/v1/courses' +ROSTER_URL = 'https://classroom.googleapis.com/v1/courses/{courseid}/students' + +def clean_data(resp_json, key, sort_key, default=None): + print("Response", resp_json) + if 'error' in resp_json: + return {'error': resp_json['error']} # Typically, resp_json['error'] == 'UNAUTHENTICATED' + if key is not None: + if key in resp_json: + resp_json = resp_json[key] + # This happens if e.g. no courses. Google seems to just return {} + # instead of {'courses': []} + else: + return default + if sort_key is not None: + resp_json.sort(key=sort_key) + print(resp_json) + return resp_json + + +async def synthetic_ajax(request, url, key=None, sort_key=None, default=None): + ''' + Stub similar to google_ajax, but grabbing data from local files. + + This is helpful for testing, but it's even more helpful since + Google is an amazingly unreliable B2B company, and this lets us + develop without relying on them. + ''' + synthetic_data = { + COURSE_URL: "static_data/courses.json", + ROSTER_URL: "static_data/students.json" + } + return clean_data(open(synthetic_data[url]).read(), default=default) + +async def google_ajax(request, url, parameters={}, key=None, sort_key=None, default=None): + ''' + Request information through Google's API + + Most requests return a dictionary with one key. If we just want + that element, set `key` to be the element of the dictionary we want + + This is usually a list. If we want to sort this, pass a function as + `sort_key` + + Note that we return error as a json object with error information, + rather than raising an exception. In most cases, we want to pass + this error back to the JavaScript client, which can then handle + loading the auth page. + ''' + async with aiohttp.ClientSession(loop=request.app.loop) as client: + async with client.get(url.format(**parameters), headers=request["auth_headers"]) as resp: + resp_json = await resp.json() + return clean_data(resp_json, key, sort_key, default=default) + +async def courselist(request): + course_list = await google_ajax( + request, + url=COURSE_URL, + key='courses', + sort_key=lambda x:x.get('name', 'ZZ'), + default=[] + ) + return course_list + + +async def courseroster(request, course_id): + roster = await google_ajax( + request, + url=ROSTER_URL, + parameters={'courseid': int(course_id)}, + key='students', + sort_key=lambda x:x.get('name', {}).get('fullName', 'ZZ'), + default=[] + ) + return roster + +async def courselist_api(request): + return aiohttp.web.json_response(await courselist(request)) + +async def courseroster_api(request): + course_id = int(request.match_info['course_id']) + return aiohttp.web.json_response(await courseroster(request, course_id)) diff --git a/webapp/static/index.html b/webapp/static/index.html index d671f59c6..f3c3132d8 100644 --- a/webapp/static/index.html +++ b/webapp/static/index.html @@ -6,7 +6,7 @@ - + diff --git a/webapp/static/modules/course.html b/webapp/static/modules/course.html new file mode 100644 index 000000000..50f6fa1d9 --- /dev/null +++ b/webapp/static/modules/course.html @@ -0,0 +1,45 @@ +
+
+
+ {{ name }} +
+
+
+
+

+

+ {{ descriptionHeading }}

+
+
+
+
+

+

+

+
+
+
+ + + diff --git a/webapp/static/modules/courses.html b/webapp/static/modules/courses.html new file mode 100644 index 000000000..7f183437a --- /dev/null +++ b/webapp/static/modules/courses.html @@ -0,0 +1,21 @@ +
+

My Courses

+
+ +
+
diff --git a/webapp/static/modules/login.html b/webapp/static/modules/login.html new file mode 100644 index 000000000..282348a9f --- /dev/null +++ b/webapp/static/modules/login.html @@ -0,0 +1,3 @@ + + + diff --git a/webapp/static/modules/navbar_loggedin.html b/webapp/static/modules/navbar_loggedin.html new file mode 100644 index 000000000..949c80fe4 --- /dev/null +++ b/webapp/static/modules/navbar_loggedin.html @@ -0,0 +1,17 @@ +
+
+ {{ user_name }} +
+
+
+ +
+
+
+
+ + Log out + +
+
+
diff --git a/webapp/static/webapp.html b/webapp/static/webapp.html new file mode 100644 index 000000000..0d750c979 --- /dev/null +++ b/webapp/static/webapp.html @@ -0,0 +1,55 @@ + + + + + + + + + + + Writing Analysis + + + +
+ + + + +
+ +
+
+
+

+ Writing Dashboard + by Piotr Mitros. Copyright + (c) 2020. Educational Testing + Service. The source + code will be released as free / open source software, + most likely + under the + AGPLv3 license. Some thoughts on privacy. +

+
+
+
+ + diff --git a/webapp/static/webapp.js b/webapp/static/webapp.js new file mode 100644 index 000000000..b72cbe90d --- /dev/null +++ b/webapp/static/webapp.js @@ -0,0 +1,136 @@ +function getCookie(cname) { + var name = cname + "="; + var decodedCookie = decodeURIComponent(document.cookie); + var ca = decodedCookie.split(';'); + for(var i = 0; i Date: Sun, 23 Aug 2020 01:13:07 +0000 Subject: [PATCH 075/651] Filesystem startup snapshot function --- webapp/filesystem_state.py | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 webapp/filesystem_state.py diff --git a/webapp/filesystem_state.py b/webapp/filesystem_state.py new file mode 100644 index 000000000..4fd34fc1b --- /dev/null +++ b/webapp/filesystem_state.py @@ -0,0 +1,55 @@ +import hashlib +import os +import subprocess + +extensions = [ + ".py", + ".js", + ".html", + ".md" +] + +filestring = """{filename}: +\thash:{hash} +\tst_mode:{st_mode} +\tst_size:{st_size} +\tst_atime:{st_atime} +\tst_mtime:{st_mtime} +\tst_ctime:{st_ctime} +""" + +def filesystem_state(): + ''' + Make a snapshot of the file system. Return a json object. Best + usage is to combine with `yaml.dump`, or `json.dump` with a + specific indent. This is helpful for knowing which version was running. + + Snapshot contains list of Python, HTML, JSON, and Markdown files, + together with their SHA hashes and modified times. It also + contains a `git` hash of the current commit. + + This ought to be enough to confirm which version of the tool is + running, and if we are running from a `git` commit (as we ought to + in production) or if changes were made since git commited. + ''' + file_info = {} + for root, dirs, files in os.walk("."): + for name in files: + for extension in extensions: + if name.endswith(extension): + filename = os.path.join(root, name) + stat = os.stat(filename) + file_info[filename] = { + "hash": hashlib.sha3_512(open(filename, "rb").read()).hexdigest(), + "st_mode": stat.st_mode, + "st_size": stat.st_size, + "st_atime": stat.st_atime, + "st_mtime": stat.st_mtime, + "st_ctime": stat.st_ctime + } + file_info['::git-head::'] = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('utf-8').strip() + return file_info + +if __name__ == '__main__': + import yaml + print(yaml.dump(filesystem_state())) From 71da683f6c8862d41d962cd26402f0b8bc703dc5 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Sun, 23 Aug 2020 01:13:40 +0000 Subject: [PATCH 076/651] Moving to main webapp on /, as well as adding docs --- webapp/aio_webapp_w.py | 4 +-- webapp/log_event.py | 48 +++++++++++++++++++++++++++++++ webapp/static/modules/course.html | 2 +- webapp/static/modules/login.html | 2 +- webapp/static/webapp.html | 4 +-- webapp/static/webapp.js | 42 ++++++++++++++++++--------- 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/webapp/aio_webapp_w.py b/webapp/aio_webapp_w.py index adafc6186..d0fe80f56 100644 --- a/webapp/aio_webapp_w.py +++ b/webapp/aio_webapp_w.py @@ -119,8 +119,8 @@ def handler(request): # Generic web-appy things app.add_routes([ -# aiohttp.web.get('/', static_file_handler("static/index.html")), - aiohttp.web.get('/', index), + aiohttp.web.get('/', static_file_handler("static/webapp.html")), +# aiohttp.web.get('/', index), aiohttp.web.get('/auth/login/{provider:google}', handler=auth_handlers.social), aiohttp.web.get('/auth/logout', handler=auth_handlers.logout), aiohttp.web.get('/auth/userinfo', handler=auth_handlers.user_info) diff --git a/webapp/log_event.py b/webapp/log_event.py index 9dc4aa237..2f46522d3 100644 --- a/webapp/log_event.py +++ b/webapp/log_event.py @@ -1,3 +1,51 @@ +# For now, we dump logs into files, crudely. +# +# We're not there yet, but we would like to create a 哈希树, or +# Merkle-tree-style structure for our log files. +# +# Or to be specific, a Merkle DAG, like git. +# +# Each item is stored under its SHA hash. Note that items are not +# guaranteed to exist. We can prune them, and leave a dangling pointer +# with just the SHA. +# +# Each event log will be structured as +# +-----------------+ +-----------------+ +# <--- Last item (SHA) | <--- Last item (SHA) | ... +# | | | | +# | Data (SHA) | | Data (SHA) | +# +-------|---------+ +--------|--------+ +# | | +# v v +# +-------+ +-------+ +# | Event | | Event | +# +-------+ +-------+ +# +# Where the top objects form a linked list (each containing a pair of +# SHA hashes, one of the previous item, and one of the associated +# event). +# +# We will then have a hierarchy, where we have lists per-document, +# documents per-student. When we run analyses, those will store the +# hashes of where in each event log we are. Likewise, with each layer +# of analysis, we'll store pointers to git hashes of code, as well as +# of intermediate files (and how those were generated). +# +# Where data is available, we can confirm we're correctly replicating +# prior tesults. +# +# The planned data structure is very similar to git, but with the +# potential for missing data without an implosion. +# +# Where data might not be available is after a FERPA, CCPA, or GDPR +# requests to change data. In those cases, we'll have dangling nodes, +# where we'll know that data used to exist, but not what it was. +# +# We might also have missing intermediate files. For example, if we do +# a dozen analyses, we'll want to know those happened and what those +# were, but we might not keep terabytes of data around (just enough to +# redo those analyses). + import datetime import inspect import json diff --git a/webapp/static/modules/course.html b/webapp/static/modules/course.html index 50f6fa1d9..6b12a88e6 100644 --- a/webapp/static/modules/course.html +++ b/webapp/static/modules/course.html @@ -6,7 +6,7 @@
-

diff --git a/webapp/static/modules/login.html b/webapp/static/modules/login.html index 282348a9f..6c8fd32f9 100644 --- a/webapp/static/modules/login.html +++ b/webapp/static/modules/login.html @@ -1,3 +1,3 @@ - + diff --git a/webapp/static/webapp.html b/webapp/static/webapp.html index 0d750c979..cd638df65 100644 --- a/webapp/static/webapp.html +++ b/webapp/static/webapp.html @@ -5,7 +5,7 @@ - + Writing Analysis @@ -17,7 +17,7 @@