From 7493fc924f1cc92a5146e1d97c7821954773354a Mon Sep 17 00:00:00 2001 From: Andrew Lucas Date: Wed, 22 Sep 2021 20:10:58 -0300 Subject: [PATCH 1/2] Add seed data to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 9569785..d0cfd14 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +app/database/seeders/data/* + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From 17ae9965a23bbafd6f021e7fb4c4ae191304cc97 Mon Sep 17 00:00:00 2001 From: Andrew Lucas Date: Wed, 22 Sep 2021 20:13:09 -0300 Subject: [PATCH 2/2] Update with production version --- alembic/versions/0b43f8b7c0bd_.py | 338 ++++++++++++++++++ alembic/versions/5908513aaf0a_.py | 114 ++++++ alembic/versions/7f9038506177_.py | 116 ++++++ alembic/versions/887880008f5d_.py | 114 ++++++ app/crud/message.py | 2 +- app/crud/paper_with_code.py | 59 +-- app/crud/submission.py | 179 +++++++--- app/crud/task.py | 2 +- app/crud/user.py | 3 +- app/database/base.py | 4 +- app/database/init_db.py | 9 +- app/database/seeders/helper.py | 17 + app/database/seeders/image_classification.py | 127 +++++++ app/database/seeders/initial_seed.py | 12 - app/database/seeders/initial_seed.sql | 56 --- app/database/seeders/machine_translation.py | 218 +++++++++++ .../seeders/named_entity_recognition.py | 115 ++++++ app/database/seeders/object_detection.py | 160 +++++++++ app/database/seeders/question_answering.py | 121 +++++++ app/models/paper.py | 4 + app/routes/login.py | 23 ++ app/routes/paper_with_code_integration.py | 27 +- app/routes/submission.py | 20 +- app/routes/user.py | 38 +- app/routes/util.py | 1 + app/schemas/message.py | 2 + app/schemas/paper.py | 1 - app/schemas/submission.py | 5 +- app/schemas/user.py | 6 +- .../email-templates/email_confirmation.html | 22 ++ .../email-templates/email_confirmation.mjml | 14 + app/utils/email-templates/new_account.html | 26 -- app/utils/email-templates/new_account.mjml | 15 - .../email-templates/submission_updates.html | 22 ++ .../email-templates/submission_updates.mjml | 13 + app/utils/email.py | 37 +- 36 files changed, 1783 insertions(+), 259 deletions(-) create mode 100644 alembic/versions/0b43f8b7c0bd_.py create mode 100644 alembic/versions/5908513aaf0a_.py create mode 100644 alembic/versions/7f9038506177_.py create mode 100644 alembic/versions/887880008f5d_.py create mode 100644 app/database/seeders/image_classification.py delete mode 100644 app/database/seeders/initial_seed.py delete mode 100644 app/database/seeders/initial_seed.sql create mode 100644 app/database/seeders/machine_translation.py create mode 100644 app/database/seeders/named_entity_recognition.py create mode 100644 app/database/seeders/object_detection.py create mode 100644 app/database/seeders/question_answering.py create mode 100644 app/utils/email-templates/email_confirmation.html create mode 100644 app/utils/email-templates/email_confirmation.mjml delete mode 100644 app/utils/email-templates/new_account.html delete mode 100644 app/utils/email-templates/new_account.mjml create mode 100644 app/utils/email-templates/submission_updates.html create mode 100644 app/utils/email-templates/submission_updates.mjml diff --git a/alembic/versions/0b43f8b7c0bd_.py b/alembic/versions/0b43f8b7c0bd_.py new file mode 100644 index 0000000..68430f6 --- /dev/null +++ b/alembic/versions/0b43f8b7c0bd_.py @@ -0,0 +1,338 @@ +"""empty message + +Revision ID: 0b43f8b7c0bd +Revises: 887880008f5d +Create Date: 2021-08-21 13:57:10.827340 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '0b43f8b7c0bd' +down_revision = '887880008f5d' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('accuracy_type', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('accuracy_type', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('accuracy_value', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('accuracy_value', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('accuracy_value', 'value', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('cpu', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('dataset', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('dataset', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('gpu', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('gpu', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('message', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('message', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('model', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('model', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'multiply_adds', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('paper', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('paper', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('submission', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('submission', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('task', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('task', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('task_dataset', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('task_dataset', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('task_dataset_accuracy_type', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('task_dataset_accuracy_type', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('tpu', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('tpu', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('tpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('tpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('user', 'created_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + op.alter_column('user', 'updated_at', + existing_type=postgresql.TIMESTAMP(), + type_=sa.DateTime(timezone=True), + existing_nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('user', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('user', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('tpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('tpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('tpu', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('tpu', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('task_dataset_accuracy_type', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('task_dataset_accuracy_type', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('task_dataset', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('task_dataset', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('task', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('task', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('submission', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('submission', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('paper', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('paper', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('model', 'multiply_adds', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('model', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('message', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('message', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('gpu', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('dataset', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('dataset', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('cpu', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('accuracy_value', 'value', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('accuracy_value', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('accuracy_value', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('accuracy_type', 'updated_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + op.alter_column('accuracy_type', 'created_at', + existing_type=sa.DateTime(timezone=True), + type_=postgresql.TIMESTAMP(), + existing_nullable=True) + # ### end Alembic commands ### diff --git a/alembic/versions/5908513aaf0a_.py b/alembic/versions/5908513aaf0a_.py new file mode 100644 index 0000000..01c5f77 --- /dev/null +++ b/alembic/versions/5908513aaf0a_.py @@ -0,0 +1,114 @@ +"""empty message + +Revision ID: 5908513aaf0a +Revises: 9f876af59e81 +Create Date: 2021-08-13 02:27:43.134868 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '5908513aaf0a' +down_revision = '9f876af59e81' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('accuracy_value', 'value', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'multiply_adds', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('tpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('tpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('tpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('tpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'multiply_adds', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('accuracy_value', 'value', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + # ### end Alembic commands ### diff --git a/alembic/versions/7f9038506177_.py b/alembic/versions/7f9038506177_.py new file mode 100644 index 0000000..f609073 --- /dev/null +++ b/alembic/versions/7f9038506177_.py @@ -0,0 +1,116 @@ +"""empty message + +Revision ID: 7f9038506177 +Revises: 5908513aaf0a +Create Date: 2021-08-13 04:15:22.478315 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7f9038506177' +down_revision = '5908513aaf0a' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('accuracy_value', 'value', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'multiply_adds', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.add_column('paper', sa.Column('pwc_link', sa.String(), nullable=True)) + op.alter_column('tpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('tpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('tpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('tpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.drop_column('paper', 'pwc_link') + op.alter_column('model', 'multiply_adds', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('accuracy_value', 'value', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + # ### end Alembic commands ### diff --git a/alembic/versions/887880008f5d_.py b/alembic/versions/887880008f5d_.py new file mode 100644 index 0000000..562f9d0 --- /dev/null +++ b/alembic/versions/887880008f5d_.py @@ -0,0 +1,114 @@ +"""empty message + +Revision ID: 887880008f5d +Revises: 7f9038506177 +Create Date: 2021-08-13 04:16:16.266566 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '887880008f5d' +down_revision = '7f9038506177' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('accuracy_value', 'value', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('model', 'multiply_adds', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('tpu', 'tdp', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + op.alter_column('tpu', 'gflops', + existing_type=sa.REAL(), + type_=sa.Float(precision=3), + existing_nullable=True) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('tpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('tpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'multiply_adds', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('model', 'hardware_burden', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('gpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'gflops', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'tdp', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('cpu', 'frequency', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + op.alter_column('accuracy_value', 'value', + existing_type=sa.Float(precision=3), + type_=sa.REAL(), + existing_nullable=True) + # ### end Alembic commands ### diff --git a/app/crud/message.py b/app/crud/message.py index 3e34392..1bc0a84 100644 --- a/app/crud/message.py +++ b/app/crud/message.py @@ -8,7 +8,7 @@ def get_multi( self, db, *, skip: int = 0, limit: int = 100, submission_id: int = None ): return db.query(Message).filter(Message.submission_id == submission_id)\ - .order_by(Message.created_at.desc()).offset(skip).limit(limit).all() + .order_by(Message.id.desc()).offset(skip).limit(limit).all() message = CRUDMessage(Message) diff --git a/app/crud/paper_with_code.py b/app/crud/paper_with_code.py index a51556b..9db7dea 100644 --- a/app/crud/paper_with_code.py +++ b/app/crud/paper_with_code.py @@ -12,6 +12,8 @@ def get_multi_model_metrics_by_identifier( limit: int, skip: int, task_dataset_identifier: str, + paper_title: str = None, + model_name: str = None ) -> List[Any]: tasks_dataset = db.query(TaskDataset).filter( @@ -27,11 +29,19 @@ def get_multi_model_metrics_by_identifier( Model.gflops.label("model_gflops"), Model.multiply_adds.label("model_multiply_adds"), Paper.identifier.label("paper_identifier"), - ).select_from(tasks_dataset)\ + Paper.title.label("paper_title"), + Paper.pwc_link.label("paper_pwc_link"), + Paper.link.label("paper_link")).select_from(tasks_dataset)\ .join(TaskDataset.models)\ .join(Model.paper)\ .filter(Paper.is_public)\ - .all() + + if model_name: + response = response.filter(Model.name.ilike(model_name)) + if paper_title: + response = response.filter(Paper.title.ilike(paper_title)) + + response = response.all() res = [] if len(response): @@ -42,52 +52,13 @@ def get_multi_model_metrics_by_identifier( 'model_name': row.model_name, 'model_hardware_burden': row.model_hardware_burden, 'model_operation_per_network_pass': row.model_gflops if row.model_gflops else row.model_multiply_adds, - 'paper_identifier': row.paper_identifier, + 'paper_title': row.paper_title, + 'paper_pwc_link': row.paper_pwc_link, + 'paper_link': row.paper_link, }) return res - def get_model_metrics_by_identifier( - self, - db: Session, *, - task_dataset_identifier: str, - model_identifier: str, - ) -> List[Any]: - - tasks_dataset = db.query(TaskDataset).filter( - TaskDataset.identifier == task_dataset_identifier - ).subquery('tasks_dataset') - - response = db.query( - tasks_dataset.c.identifier.label("tasks_dataset_identifier"), - Model.identifier.label("model_identifier"), - Model.name.label("model_name"), - Model.hardware_burden.label("model_hardware_burden"), - Model.gflops.label("model_gflops"), - Model.multiply_adds.label("model_multiply_adds"), - Model.extra_training_time.label("model_extra_training_time"), - Paper.identifier.label("paper_identifier"), - ).select_from(tasks_dataset)\ - .join(TaskDataset.models)\ - .join(Model.paper)\ - .filter(Paper.is_public)\ - .filter(Model.identifier == model_identifier)\ - .all() - - res = None - if len(response): - logging.error(response) - res = { - 'tasks_dataset_identifier': response[0].tasks_dataset_identifier, - 'model_identifier': response[0].model_identifier, - 'model_name': response[0].model_name, - 'model_hardware_burden': response[0].model_hardware_burden, - 'model_operation_per_network_pass': response[0].model_gflops if response[0].model_gflops else response[0].model_multiply_adds, - 'paper_identifier': response[0].paper_identifier, - } - - return res - paper_with_code = CRUDTask() diff --git a/app/crud/submission.py b/app/crud/submission.py index e46f4ac..3be7747 100644 --- a/app/crud/submission.py +++ b/app/crud/submission.py @@ -1,3 +1,4 @@ +from app.utils.email import send_submission_updates_email import json from app.models.submission import StatusEnum from app.models.task_dataset_accuracy_type import TaskDatasetAccuracyType @@ -16,36 +17,88 @@ def checkDiff(oldSubmission, newSubmission): messages = [] - print(type(oldSubmission), flush=True) - print(oldSubmission, flush=True) - if oldSubmission.title != newSubmission.title: - messages.append('paper title changed to {}'.format(newSubmission.title)) - - if oldSubmission.link != newSubmission.link: - messages.append('paper link changed to {}'.format(newSubmission.link)) - - if oldSubmission.code_link != newSubmission.code_link: - messages.append('paper code link changed to {}'.format( - newSubmission.code_link)) - - if oldSubmission.publication_date != newSubmission.publication_date: - messages.append('paper publication date changed to {}'.format( - newSubmission.publication_date)) - - if oldSubmission.authors != newSubmission.authors: - messages.append('paper authors changed to {}'.format( - ' '.join(newSubmission.authors))) - - oldModels = [model.json() for model in newSubmission.models] - newModels = [model.json() for model in oldSubmission.models] - if len(oldSubmission.models) > len(newSubmission.models): - messages.append('{} model deleted'.format( - len(oldSubmission.models) - len(newSubmission.models))) - elif len(oldSubmission.models) < len(newSubmission.models): - messages.append('new model added') - - elif oldModels != newModels: - messages.append('models updated') + oldSubmission = oldSubmission.dict() + newSubmission = newSubmission.dict() + + mismatch = {key for key in oldSubmission.keys( + ) & newSubmission if oldSubmission[key] != newSubmission[key]} + + if oldSubmission['title'] != newSubmission['title']: + messages.append('paper\'s title was changed from "{}" to "{}"'.format( + oldSubmission['title'], newSubmission['title'])) + + if oldSubmission['link'] != newSubmission['link']: + messages.append('paper\'s link was changed from "{}" to "{}"'.format( + oldSubmission['link'], newSubmission['link'])) + + if oldSubmission['code_link'] != newSubmission['code_link']: + messages.append('paper\'s code link was changed from "{}" to "{}"'.format( + oldSubmission['code_link'], newSubmission['code_link'])) + + if oldSubmission['publication_date'] != newSubmission['publication_date']: + messages.append('paper\'s publication date was changed from "{}" to "{}"'.format( + oldSubmission['publication_date'], newSubmission['publication_date'])) + + if oldSubmission['authors'] != newSubmission['authors']: + messages.append('paper\'s authors were changed from {} to {}'.format( + ', '.join(oldSubmission['authors']), ', '.join(newSubmission['authors']))) + + if 'models' in mismatch: + + removed_models = [x for x in oldSubmission['models'] if x['name'] + not in [model['name'] for model in newSubmission['models']]] + + for removed in removed_models: + messages.append('{} model removed'.format(removed['name'])) + + added_models = [x for x in newSubmission['models'] if x['name'] + not in [model['name'] for model in oldSubmission['models']]] + + for added in added_models: + messages.append('{} model added'.format(added['name'])) + + possibly_modified_models = [x for x in oldSubmission['models'] if x['name'] not in [ + model['name'] for model in removed_models]] + + for possibly_modified_model in possibly_modified_models: + + model_modified = next( + (item for item in newSubmission['models'] if item['name'] == possibly_modified_model['name']), None) + + if model_modified: + + mismatch = {key for key in possibly_modified_model.keys( + ) & model_modified if possibly_modified_model[key] != model_modified[key]} + + for field in mismatch: + if field == 'accuracies': + removed_accuracies = [x for x in possibly_modified_model['accuracies'] if x['accuracy_type'] + not in [model['accuracy_type'] for model in model_modified['accuracies']]] + + for removed in removed_accuracies: + messages.append('{} accuracy was removed from {}'.format( + removed['accuracy_type'], model_modified['name'])) + + added_accuracies = [x for x in model_modified['accuracies'] if x['accuracy_type'] + not in [model['accuracy_type'] for model in possibly_modified_model['accuracies']]] + + for added in added_accuracies: + messages.append('{} accuracy was added to {}'.format( + added['accuracy_type'], model_modified['name'])) + + possibly_modified_accruacies = [x for x in possibly_modified_model['accuracies'] if x['accuracy_type'] not in [ + accuracy['accuracy_type'] for accuracy in removed_accuracies]] + + for accuracy_possibly_modified in possibly_modified_accruacies: + accuracy_modified = next( + (item for item in model_modified['accuracies'] if item['accuracy_type'] == accuracy_possibly_modified['accuracy_type']), None) + + if accuracy_modified and accuracy_possibly_modified['value'] != accuracy_modified['value']: + messages.append('{} accuracy was changed from "{}" to "{}" on {}'.format( + model_modified['name'], accuracy_modified['accuracy_type'], accuracy_possibly_modified['value'], accuracy_modified['value'])) + else: + messages.append('{}\'s {} was changed from "{}" to "{}"'.format(model_modified['name'], field.replace( + '_', ' '), possibly_modified_model[field], model_modified[field])) return messages @@ -54,11 +107,11 @@ def checkFields(db, obj_in): for model_data in obj_in.models: # Check if task exists if not db.query(Task).filter(Task.name.ilike(model_data.task)).first(): - messages.append("new task requested: {}".format(model_data.task)) + messages.append('new task requested: "{}"'.format(model_data.task)) # Check if dataset exists if not db.query(Dataset).filter(Dataset.name.ilike(model_data.dataset)).first(): - messages.append("new dataset requested: {}".format(model_data.dataset)) + messages.append('new dataset requested: "{}"'.format(model_data.dataset)) # Check if association between task and dataset exists if not db.query(TaskDataset).filter(TaskDataset.identifier == "{}-on-{}".format( @@ -66,25 +119,25 @@ def checkFields(db, obj_in): slugify(model_data.dataset, max_length=45, word_boundary=True), )).first(): messages.append( - "new association beetween task and dataset requested: {} on {}".format( + 'new association beetween task and dataset requested: "{}" on "{}"'.format( model_data.task, model_data.dataset)) if model_data.cpu and not db.query(Cpu).filter( Cpu.name.ilike(model_data.cpu)).first(): - messages.append("new cpu requested: {}".format(model_data.cpu)) + messages.append('new cpu requested: "{}"'.format(model_data.cpu)) if not db.query(Gpu).filter( Gpu.name.ilike(model_data.gpu)).first(): - messages.append("new gpu requested: {}".format(model_data.gpu)) + messages.append('new gpu requested: "{}"'.format(model_data.gpu)) if model_data.tpu and not db.query(Tpu).filter( Tpu.name.ilike(model_data.tpu)).first(): - messages.append("new tpu requested: {}".format(model_data.tpu)) + messages.append('new tpu requested: "{}"'.format(model_data.tpu)) for accuracy in model_data.accuracies: if not db.query(AccuracyType).filter( AccuracyType.name.ilike(accuracy.accuracy_type)).first(): - messages.append("new accuracy type requested: {}, with value {}".format( + messages.append('new accuracy type requested: "{}" with value "{}"'.format( accuracy.accuracy_type, accuracy.value)) return messages @@ -105,16 +158,21 @@ def calculate_hardware_burden(model): class CRUDSubmission(CRUDBase[Submission, SubmissionData, SubmissionData]): def get_multi( - self, db, *, skip: int = 0, limit: int = 100, q: str = None, owner_id + self, db, *, skip: int = 0, limit: int = 100, q: str = None, owner_id, status ): - submissions = db.query(Submission) + submissions = db.query(Submission).filter(Submission.data.isnot(None)) if owner_id: submissions = submissions.filter(Submission.owner_id == owner_id) if q: submissions = submissions.filter( Submission.data.as_json()['title'].as_string().ilike("%{}%".format(q))) + if status: + submissions = submissions.filter(Submission.status == status) - return submissions.offset(skip).limit(limit).all() + return { + 'items': submissions.offset(skip).limit(limit).all(), + 'total': submissions.count() + } def create(self, db, *, obj_in: SubmissionData, current_user): submission = Submission( @@ -165,8 +223,12 @@ def update(self, db, *, db_obj: Submission, obj_in: SubmissionData, current_user detail="Error on update submission") def update_status(self, db, *, - db_obj: Submission, status: StatusEnum, current_user): + db_obj: Submission, + status: StatusEnum, + current_user, + background_tasks): submission = db_obj + if submission.status == StatusEnum.approved: raise HTTPException( status_code=400, @@ -176,17 +238,35 @@ def update_status(self, db, *, if submission.paper: submission.paper.is_public = True else: - return self.process_submission(db, submission=submission, - current_user=current_user) - - elif status == StatusEnum.declined: + updated_submission = self.process_submission(db, submission=submission, + current_user=current_user) + message = 'We are writing to you just to let you know that your submission regarding the parper "{}" was approved.'.format( + submission.data.title) + background_tasks.add_task( + send_submission_updates_email, email_to=submission.owner.email, + message=message) + return updated_submission + + elif status == StatusEnum.declined and submission.status != StatusEnum.declined: submission.status = StatusEnum.declined if submission.paper: submission.paper.is_public = False - # TODO: send declined email - elif status == StatusEnum.need_information: + message = 'We are writing to you just to let you know that your submission regarding the parper "{}" was declined.'.format( + submission.data.title) + db.add(Message(submission=submission, body='submission status changed to "declined" by {}'.format( + ' '.join([current_user.first_name, current_user.last_name])))) + background_tasks.add_task( + send_submission_updates_email, email_to=submission.owner.email, + message=message) + elif status == StatusEnum.need_information and submission.status != StatusEnum.need_information: submission.status = StatusEnum.need_information - # TODO: send need_information email + message = 'We are writing to you just to let you know that your submission regarding the parper "{}" is under review and that some additional information has been requested.'.format( + submission.data.title) + db.add(Message(submission=submission, body='submission status changed to "need information" by {}'.format( + ' '.join([current_user.first_name, current_user.last_name])))) + background_tasks.add_task( + send_submission_updates_email, email_to=submission.owner.email, + message=message) submission.reviewer = current_user db.add(submission) db.commit() @@ -284,7 +364,8 @@ def process_submission(self, db, *, submission: Submission, current_user): paper.models.append(model) db.add(paper) - db.add(Message(submission=submission, body="submission approved")) + db.add(Message(submission=submission, body='submission status changed to "approved" by {}'.format( + ' '.join([current_user.first_name, current_user.last_name])))) db.commit() return submission diff --git a/app/crud/task.py b/app/crud/task.py index 1354011..e7c5380 100644 --- a/app/crud/task.py +++ b/app/crud/task.py @@ -435,7 +435,7 @@ def get_models_csv( 'task_name': row.task_name, 'dataset_name': row.dataset_name, 'model_id': row.model_id, - 'paper_publication_date': row.paper_publication_date.year, + 'paper_publication_date': row.paper_publication_date and row.paper_publication_date.year, 'paper_title': row.paper_title, 'paper_link': row.paper_link, 'paper_code_link': row.paper_code_link, diff --git a/app/crud/user.py b/app/crud/user.py index 08f5beb..ad23774 100644 --- a/app/crud/user.py +++ b/app/crud/user.py @@ -13,13 +13,14 @@ class CRUDUser(CRUDBase[User, UserCreate, UserUpdate]): def get_by_email(self, db: Session, *, email: str) -> Optional[User]: return db.query(User).filter(User.email == email).first() - def create(self, db: Session, *, obj_in: UserCreate) -> User: + def create(self, db: Session, *, obj_in: UserCreate, is_active=False) -> User: db_obj = User( email=obj_in.email, hashed_password=get_password_hash(obj_in.password), first_name=obj_in.first_name, last_name=obj_in.last_name, role=obj_in.role, + is_active=is_active ) db.add(db_obj) db.commit() diff --git a/app/database/base.py b/app/database/base.py index 0640315..68c4956 100644 --- a/app/database/base.py +++ b/app/database/base.py @@ -8,8 +8,8 @@ class Base: id: Any __name__: str - created_at = Column(DateTime, default=func.now()) - updated_at = Column(DateTime, default=func.now(), onupdate=func.now()) + created_at = Column(DateTime(timezone=True), default=func.now()) + updated_at = Column(DateTime(timezone=True), default=func.now(), onupdate=func.now()) # Generate __tablename__ automatically @declared_attr diff --git a/app/database/init_db.py b/app/database/init_db.py index 90bfd11..552becc 100644 --- a/app/database/init_db.py +++ b/app/database/init_db.py @@ -5,7 +5,8 @@ from app.database import base # noqa: F401 from app.database.seeders import ( - cpu, gpu, initial_seed, tpu) + cpu, gpu, tpu, image_classification, machine_translation, named_entity_recognition, + object_detection, question_answering) # make sure all SQL Alchemy models are imported (app.db.base) before initializing DB # otherwise, SQL Alchemy might fail to initialize relationships properly @@ -29,4 +30,8 @@ def init_db(db: Session) -> None: cpu.seed() gpu.seed() tpu.seed() - initial_seed.seed() + image_classification.seed() + machine_translation.seed() + named_entity_recognition.seed() + object_detection.seed() + question_answering.seed() diff --git a/app/database/seeders/helper.py b/app/database/seeders/helper.py index 2da6546..e896714 100644 --- a/app/database/seeders/helper.py +++ b/app/database/seeders/helper.py @@ -31,3 +31,20 @@ def executeSQLfromFile(db, filename): logging.error(e) finally: sql_command = '' + + +def calculate_hardware_burden(model): + hw = 0 + if model.cpu and model.cpu.gflops and model.number_of_cpus: + hw = hw + model.cpu.gflops * model.number_of_cpus + if model.gpu and model.gpu.gflops and model.number_of_gpus: + hw = hw + model.gpu.gflops * model.number_of_gpus + if model.tpu and model.tpu.gflops and model.number_of_tpus: + hw = hw + model.tpu.gflops * model.number_of_tpus + + if model.training_time: + hw = hw * model.training_time + else: + hw = None + + return hw if hw else None diff --git a/app/database/seeders/image_classification.py b/app/database/seeders/image_classification.py new file mode 100644 index 0000000..ceda849 --- /dev/null +++ b/app/database/seeders/image_classification.py @@ -0,0 +1,127 @@ +from datetime import datetime + +from fastapi.encoders import jsonable_encoder +from app import models, schemas +import csv +from app.database.session import SessionLocal +from app.database.seeders.helper import parseFloat, parseInt, calculate_hardware_burden +import os + + +def check_none(value): + if isinstance(value, int) or isinstance(value, float): + return value + return 0 + + +def get_date(year): + if parseInt(year): + return datetime(year=int(year), month=6, day=15) + return None + + +def seed() -> None: + db = SessionLocal() + + task = models.Task(name='Image Classification', identifier='image-classification', + description='Image Classification is a fundamental task that attempts to comprehend an entire image as a whole. The goal is to classify the image by assigning it to a specific label. Typically, Image Classification refers to images in which only one object appears and is analyzed. In contrast, object detection involves both classification and localization tasks, and is used to analyze more realistic cases in which multiple objects may exist in an image.', + image='https://computerprogress.xyz/image/image-classification.svg') + + dataset = models.Dataset(name='Imagenet', identifier='imagenet') + + top1 = models.AccuracyType(name='TOP 1') + top5 = models.AccuracyType(name='TOP 5') + + task_dataset_accuracy_type_top1 = models.TaskDatasetAccuracyType( + required=True, main=True, accuracy_type=top1) + task_dataset_accuracy_type_top5 = models.TaskDatasetAccuracyType( + required=False, main=False, accuracy_type=top5) + + task_dataset = models.TaskDataset(task=task, dataset=dataset) + + task_dataset.accuracy_types.append(task_dataset_accuracy_type_top1) + task_dataset.accuracy_types.append(task_dataset_accuracy_type_top5) + + if os.path.exists('app/database/seeders/data/PaperWithCode_Integration_Data_CV_Image_Classification_Imagenet.csv'): + with open('app/database/seeders/data/PaperWithCode_Integration_Data_CV_Image_Classification_Imagenet.csv', mode='r') as csv_file: + csv_reader = csv.DictReader(csv_file) + papers = {} + for row in csv_reader: + if papers.get(row.get('title')): + papers[row.get('title')].append(row) + else: + papers[row.get('title')] = [row] + + for _, p in papers.items(): + paper = { + 'title': p[0].get('title'), + 'link': p[0].get('paper_url'), + 'pwc_link': p[0].get('pwc_url'), + 'code_link': p[0].get('code_link'), + 'publication_date': get_date(p[0].get('year')), + 'authors': p[0].get('authors'), + } + paper = models.Paper(**paper) + paper.submission = models.Submission(status='approved') + + count = 0 + for m in p: + count += 1 + + model = { + 'name': m.get('method').strip() if m.get('method') else None, + 'training_time': parseInt(m.get('time_sec')), + 'gflops': + (parseFloat(m.get('flops')) / 10e9) if ( + parseFloat(m.get('flops'))) else None, + 'epochs': parseInt(m.get('epochs')), + 'number_of_parameters': parseInt(m.get('number_of_parameters')), + 'multiply_adds': + (parseFloat(m.get('multiply_adds')) / 10e9) if ( + parseFloat(m.get('multiply_adds'))) else None, + 'number_of_cpus': parseInt(m.get('#cpu')), + 'number_of_gpus': parseInt(m.get('#gpu')), + 'number_of_tpus': parseInt(m.get('#tpu')), + } + try: + + model = jsonable_encoder(model) + schemas.Model(**model) + except Exception: + + continue + model = models.Model(**model) + + model.paper = paper + + if m.get('cpu'): + model.cpu = db.query(models.Cpu).filter( + models.Cpu.name.ilike(m.get('cpu'))).first() + if m.get('gpu'): + model.gpu = db.query(models.Gpu).filter( + models.Gpu.name.ilike(m.get('gpu'))).first() + if m.get('tpu'): + model.tpu = db.query(models.Tpu).filter( + models.Tpu.name.ilike(m.get('tpu'))).first() + + hardware_burden = calculate_hardware_burden(model) + model.hardware_burden = hardware_burden if hardware_burden != 0 else None + + top1_accuracy = 100-parseFloat(m.get('top1_error')) if parseFloat( + m.get('top1_error')) else parseFloat(m.get('top1_error')) + top5_accuracy = 100-parseFloat(m.get('top5_error')) if parseFloat( + m.get('top5_error')) else parseFloat(m.get('top5_error')) + + models.AccuracyValue( + value=top1_accuracy, + accuracy_type=top1, + model=model + ) + models.AccuracyValue( + value=top5_accuracy, + accuracy_type=top5, + model=model + ) + task_dataset.models.append(model) + db.add(task_dataset) + db.commit() diff --git a/app/database/seeders/initial_seed.py b/app/database/seeders/initial_seed.py deleted file mode 100644 index d1f7934..0000000 --- a/app/database/seeders/initial_seed.py +++ /dev/null @@ -1,12 +0,0 @@ -from app.database.seeders.helper import executeSQLfromFile - -from app.database.session import SessionLocal -import os - - -def seed() -> None: - db = SessionLocal() - executeSQLfromFile(db, 'app/database/seeders/initial_seed.sql') - if os.path.isfile('app/database/seeders/data_seed.sql'): - executeSQLfromFile(db, 'app/database/seeders/data_seed.sql') - db.commit() diff --git a/app/database/seeders/initial_seed.sql b/app/database/seeders/initial_seed.sql deleted file mode 100644 index 3d42c1f..0000000 --- a/app/database/seeders/initial_seed.sql +++ /dev/null @@ -1,56 +0,0 @@ --- tasks -insert into "task" ("created_at", "description", "id", "identifier", "image", "name", "updated_at") values ('2021-07-02 02:08:29.864713', 'Image Classification is a fundamental task that attempts to comprehend an entire image as a whole. The goal is to classify the image by assigning it to a specific label. Typically, Image Classification refers to images in which only one object appears and is analyzed. In contrast, object detection involves both classification and localization tasks, and is used to analyze more realistic cases in which multiple objects may exist in an image.', 1, 'image-classification', 'https://computerprogress.xyz/image/image-classification.svg', 'Image Classification', '2021-07-02 02:08:29.864713'); -insert into "task" ("created_at", "description", "id", "identifier", "image", "name", "updated_at") values ('2021-07-02 02:08:30.399504', 'Named entity recognition (NER) is the task of tagging entities in text with their corresponding type. Approaches typically use BIO notation, which differentiates the beginning (B) and the inside (I) of entities. O is used for non-entity tokens.', 2, 'named-entity-recognition', 'https://computerprogress.xyz/image/named-entity-recognition.svg', 'Named Entity Recognition', '2021-07-02 02:08:30.399504'); -insert into "task" ("created_at", "description", "id", "identifier", "image", "name", "updated_at") values ('2021-07-02 02:08:30.547009', 'Object detection is the task of detecting instances of objects of a certain class within an image. The state-of-the-art methods can be categorized into two main types: one-stage methods and two stage-methods. One-stage methods prioritize inference speed, and example models include YOLO, SSD and RetinaNet. Two-stage methods prioritize detection accuracy, and example models include Faster R-CNN, Mask R-CNN and Cascade R-CNN. The most popular benchmark is the MSCOCO dataset. Models are typically evaluated according to a Mean Average Precision metric.', 3, 'object-detection', 'https://computerprogress.xyz/image/object-detection.svg', 'Object Detection', '2021-07-02 02:08:30.547009'); -insert into "task" ("created_at", "description", "id", "identifier", "image", "name", "updated_at") values ('2021-07-02 02:08:30.971315', 'Question Answering is the task of answering questions (typically reading comprehension questions), but abstaining when presented with a question that cannot be answered based on the provided context.', 4, 'question-answering', 'https://computerprogress.xyz/image/question-answering.svg', 'Question Answering', '2021-07-02 02:08:30.971315'); -insert into "task" ("created_at", "description", "id", "identifier", "image", "name", "updated_at") values ('2021-07-02 02:08:31.147218', 'Machine translation is the task of translating a sentence in a source language to a different target language.', 5, 'machine-translation', 'https://computerprogress.xyz/image/machine-translation.svg', 'Machine Translation', '2021-07-02 02:08:31.147218'); - --- datasets -insert into "dataset" ("created_at", "description", "id", "identifier", "image", "name", "source", "updated_at") values ('2021-07-02 02:08:29.864713', NULL, 1, 'imagenet', NULL, 'Imagenet', NULL, '2021-07-02 02:08:29.864713'); -insert into "dataset" ("created_at", "description", "id", "identifier", "image", "name", "source", "updated_at") values ('2021-07-02 02:08:30.399504', NULL, 2, 'conll-2003', NULL, 'Conll 2003', NULL, '2021-07-02 02:08:30.399504'); -insert into "dataset" ("created_at", "description", "id", "identifier", "image", "name", "source", "updated_at") values ('2021-07-02 02:08:30.547009', NULL, 3, 'ms-coco', NULL, 'MS COCO', NULL, '2021-07-02 02:08:30.547009'); -insert into "dataset" ("created_at", "description", "id", "identifier", "image", "name", "source", "updated_at") values ('2021-07-02 02:08:30.971315', NULL, 4, 'squad11', NULL, 'SQuAD 1.1', NULL, '2021-07-02 02:08:30.971315'); -insert into "dataset" ("created_at", "description", "id", "identifier", "image", "name", "source", "updated_at") values ('2021-07-02 02:08:31.147218', NULL, 5, 'wmt2014-en-ge', NULL, 'WMT2014 English-German', NULL, '2021-07-02 02:08:31.147218'); -insert into "dataset" ("created_at", "description", "id", "identifier", "image", "name", "source", "updated_at") values ('2021-07-02 02:08:31.312437', NULL, 6, 'wmt2014-en-fr', NULL, 'WMT2014 English-French', NULL, '2021-07-02 02:08:31.312437'); - --- accuracy types -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:29.864713', NULL, 1, 'TOP 1', '2021-07-02 02:08:29.864713'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:29.864713', NULL, 2, 'TOP 5', '2021-07-02 02:08:29.864713'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.399504', NULL, 3, 'F1', '2021-07-02 02:08:30.399504'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.547009', NULL, 4, 'BOX AP', '2021-07-02 02:08:30.547009'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.547009', NULL, 5, 'AP50', '2021-07-02 02:08:30.547009'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.547009', NULL, 6, 'AP75', '2021-07-02 02:08:30.547009'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.547009', NULL, 7, 'APS', '2021-07-02 02:08:30.547009'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.547009', NULL, 8, 'APM', '2021-07-02 02:08:30.547009'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.547009', NULL, 9, 'APL', '2021-07-02 02:08:30.547009'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.971315', NULL, 10, 'F1', '2021-07-02 02:08:30.971315'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:30.971315', NULL, 11, 'EM', '2021-07-02 02:08:30.971315'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:31.147218', NULL, 12, 'BLEU score', '2021-07-02 02:08:31.147218'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:31.147218', NULL, 13, 'SacreBLEU', '2021-07-02 02:08:31.147218'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:31.312437', NULL, 14, 'BLEU score', '2021-07-02 02:08:31.312437'); -insert into "accuracy_type" ("created_at", "description", "id", "name", "updated_at") values ('2021-07-02 02:08:31.312437', NULL, 15, 'SacreBLEU', '2021-07-02 02:08:31.312437'); - --- task dataset association -insert into "task_dataset" ("created_at", "dataset_id", "id", "identifier", "task_id", "updated_at") values ('2021-07-02 02:08:29.864713', 1, 1, 'image-classification-on-imagenet', 1, '2021-07-02 02:08:29.864713'); -insert into "task_dataset" ("created_at", "dataset_id", "id", "identifier", "task_id", "updated_at") values ('2021-07-02 02:08:30.399504', 2, 2, 'named-entity-recognition-on-conll-2003', 2, '2021-07-02 02:08:30.399504'); -insert into "task_dataset" ("created_at", "dataset_id", "id", "identifier", "task_id", "updated_at") values ('2021-07-02 02:08:30.547009', 3, 3, 'object-detection-on-ms-coco', 3, '2021-07-02 02:08:30.547009'); -insert into "task_dataset" ("created_at", "dataset_id", "id", "identifier", "task_id", "updated_at") values ('2021-07-02 02:08:30.971315', 4, 4, 'question-answering-on-squad11', 4, '2021-07-02 02:08:30.971315'); -insert into "task_dataset" ("created_at", "dataset_id", "id", "identifier", "task_id", "updated_at") values ('2021-07-02 02:08:31.147218', 5, 5, 'machine-translation-on-wmt2014-en-ge', 5, '2021-07-02 02:08:31.147218'); -insert into "task_dataset" ("created_at", "dataset_id", "id", "identifier", "task_id", "updated_at") values ('2021-07-02 02:08:31.312437', 6, 6, 'machine-translation-on-wmt2014-en-fr', 5, '2021-07-02 02:08:31.312437'); - --- task dataset accuracy type association -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (1, '2021-07-02 02:08:29.864713', 1, true, true, 1, '2021-07-02 02:08:29.864713'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (2, '2021-07-02 02:08:29.864713', 2, false, false, 1, '2021-07-02 02:08:29.864713'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (3, '2021-07-02 02:08:30.399504', 3, true, true, 2, '2021-07-02 02:08:30.399504'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (4, '2021-07-02 02:08:30.547009', 4, true, true, 3, '2021-07-02 02:08:30.547009'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (5, '2021-07-02 02:08:30.547009', 5, false, false, 3, '2021-07-02 02:08:30.547009'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (6, '2021-07-02 02:08:30.547009', 6, false, false, 3, '2021-07-02 02:08:30.547009'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (7, '2021-07-02 02:08:30.547009', 7, false, false, 3, '2021-07-02 02:08:30.547009'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (8, '2021-07-02 02:08:30.547009', 8, false, false, 3, '2021-07-02 02:08:30.547009'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (9, '2021-07-02 02:08:30.547009', 9, false, false, 3, '2021-07-02 02:08:30.547009'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (10, '2021-07-02 02:08:30.971315', 10, true, true, 4, '2021-07-02 02:08:30.971315'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (11, '2021-07-02 02:08:30.971315', 11, false, false, 4, '2021-07-02 02:08:30.971315'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (12, '2021-07-02 02:08:31.147218', 12, true, true, 5, '2021-07-02 02:08:31.147218'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (13, '2021-07-02 02:08:31.147218', 13, false, false, 5, '2021-07-02 02:08:31.147218'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (14, '2021-07-02 02:08:31.312437', 14, true, true, 6, '2021-07-02 02:08:31.312437'); -insert into "task_dataset_accuracy_type" ("accuracy_type_id", "created_at", "id", "main", "required", "task_dataset_id", "updated_at") values (15, '2021-07-02 02:08:31.312437', 15, false, false, 6, '2021-07-02 02:08:31.312437'); diff --git a/app/database/seeders/machine_translation.py b/app/database/seeders/machine_translation.py new file mode 100644 index 0000000..0684ec1 --- /dev/null +++ b/app/database/seeders/machine_translation.py @@ -0,0 +1,218 @@ +from datetime import datetime +import os + +from fastapi.encoders import jsonable_encoder +from app import models, schemas +import csv +from app.database.session import SessionLocal +from app.database.seeders.helper import parseFloat, parseInt, calculate_hardware_burden + + +def check_none(value): + if isinstance(value, int) or isinstance(value, float): + return value + return 0 + + +def get_date(year): + if parseInt(year): + return datetime(year=int(year), month=6, day=15) + return None + + +def seed() -> None: + db = SessionLocal() + + task = models.Task(name='Machine Translation', identifier='machine-translation', + description='Machine translation is the task of translating a sentence in a source language to a different target language.', + image='https://computerprogress.xyz/image/machine-translation.svg') + + dataset_en_ge = models.Dataset( + name='WMT2014 English-German', identifier='wmt2014-en-ge') + dataset_en_fr = models.Dataset( + name='WMT2014 English-French', identifier='wmt2014-en-fr') + + BLEU = models.AccuracyType(name='BLEU score') + SACREBLEU = models.AccuracyType(name='SacreBLEU') + + task_dataset_en_ge = models.TaskDataset(task=task, dataset=dataset_en_ge) + task_dataset_en_fr = models.TaskDataset(task=task, dataset=dataset_en_fr) + + task_dataset_en_ge.accuracy_types.append( + models.TaskDatasetAccuracyType( + required=True, + main=True, + accuracy_type=BLEU + )) + task_dataset_en_ge.accuracy_types.append( + models.TaskDatasetAccuracyType( + required=False, + main=False, + accuracy_type=SACREBLEU)) + + task_dataset_en_fr.accuracy_types.append( + models.TaskDatasetAccuracyType( + required=True, + main=True, + accuracy_type=BLEU)) + task_dataset_en_fr.accuracy_types.append( + models.TaskDatasetAccuracyType( + required=False, + main=False, + accuracy_type=SACREBLEU)) + + if os.path.exists('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Machine_Translation_WMT2014_English_German.csv'): + with open('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Machine_Translation_WMT2014_English_German.csv', mode='r') as csv_file: + csv_reader = csv.DictReader(csv_file) + papers = {} + for row in csv_reader: + if papers.get(row.get('title')): + papers[row.get('title')].append(row) + else: + papers[row.get('title')] = [row] + + for _, p in papers.items(): + paper = { + 'title': p[0].get('title'), + 'link': p[0].get('paper_url'), + 'pwc_link': p[0].get('pwc_url'), + 'code_link': p[0].get('code_link'), + 'publication_date': get_date(p[0].get('year')), + 'authors': p[0].get('authors'), + } + paper = models.Paper(**paper) + paper.submission = models.Submission(status='approved') + + count = 0 + for m in p: + count += 1 + + model = { + 'name': m.get('method').strip() if m.get('method') else None, + 'training_time': parseInt(m.get('time_sec')), + 'gflops': + (parseFloat(m.get('flops')) / 10e9) if ( + parseFloat(m.get('flops'))) else None, + 'epochs': parseInt(m.get('#epochs')), + 'number_of_parameters': parseInt(m.get('number_of_parameters')), + 'multiply_adds': + (parseFloat(m.get('multiadds')) / 10e9) if ( + parseFloat(m.get('multiadds'))) else None, + 'number_of_cpus': parseInt(m.get('#cpu')), + 'number_of_gpus': parseInt(m.get('#gpu')), + 'number_of_tpus': parseInt(m.get('#tpu')), + } + try: + + model = jsonable_encoder(model) + schemas.Model(**model) + except Exception: + + continue + model = models.Model(**model) + + model.paper = paper + + if m.get('cpu'): + model.cpu = db.query(models.Cpu).filter( + models.Cpu.name == m.get('cpu')).first() + if m.get('gpu'): + model.gpu = db.query(models.Gpu).filter( + models.Gpu.name == m.get('gpu')).first() + if m.get('tpu'): + model.tpu = db.query(models.Tpu).filter( + models.Tpu.name == m.get('tpu')).first() + + hardware_burden = calculate_hardware_burden(model) + model.hardware_burden = hardware_burden if hardware_burden != 0 else None + + models.AccuracyValue( + value=parseFloat(m.get('BLEU')), + accuracy_type=BLEU, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('SACREBLEU')), + accuracy_type=SACREBLEU, + model=model + ) + task_dataset_en_ge.models.append(model) + if os.path.exists('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Machine_Translation_WMT2014_English_French.csv'): + with open('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Machine_Translation_WMT2014_English_French.csv', mode='r') as csv_file: + csv_reader = csv.DictReader(csv_file) + papers = {} + for row in csv_reader: + if papers.get(row.get('title')): + papers[row.get('title')].append(row) + else: + papers[row.get('title')] = [row] + + for _, p in papers.items(): + paper = { + 'title': p[0].get('title'), + 'link': p[0].get('paper_url'), + 'pwc_link': p[0].get('pwc_url'), + 'code_link': p[0].get('code_link'), + 'publication_date': get_date(p[0].get('year')), + 'authors': p[0].get('authors'), + } + paper = models.Paper(**paper) + paper.submission = models.Submission(status='approved') + + count = 0 + for m in p: + count += 1 + + model = { + 'name': m.get('method').strip() if m.get('method') else None, + 'training_time': parseInt(m.get('time_sec')), + 'gflops': + (parseFloat(m.get('flops')) / 10e9) if ( + parseFloat(m.get('flops'))) else None, + 'epochs': parseInt(m.get('#epochs')), + 'number_of_parameters': parseInt(m.get('number_of_parameters')), + 'multiply_adds': + (parseFloat(m.get('multiadds')) / 10e9) if ( + parseFloat(m.get('multiadds'))) else None, + 'number_of_cpus': parseInt(m.get('#cpu')), + 'number_of_gpus': parseInt(m.get('#gpu')), + 'number_of_tpus': parseInt(m.get('#tpu')), + } + try: + + model = jsonable_encoder(model) + schemas.Model(**model) + except Exception: + + continue + model = models.Model(**model) + + model.paper = paper + + if m.get('cpu'): + model.cpu = db.query(models.Cpu).filter( + models.Cpu.name.ilike(m.get('cpu'))).first() + if m.get('gpu'): + model.gpu = db.query(models.Gpu).filter( + models.Gpu.name.ilike(m.get('gpu'))).first() + if m.get('tpu'): + model.tpu = db.query(models.Tpu).filter( + models.Tpu.name.ilike(m.get('tpu'))).first() + + hardware_burden = calculate_hardware_burden(model) + model.hardware_burden = hardware_burden if hardware_burden != 0 else None + + models.AccuracyValue( + value=parseFloat(m.get('BLEU')), + accuracy_type=BLEU, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('SACREBLEU')), + accuracy_type=SACREBLEU, + model=model + ) + task_dataset_en_fr.models.append(model) + db.add(task_dataset_en_fr) + db.add(task_dataset_en_ge) + db.commit() diff --git a/app/database/seeders/named_entity_recognition.py b/app/database/seeders/named_entity_recognition.py new file mode 100644 index 0000000..025aadc --- /dev/null +++ b/app/database/seeders/named_entity_recognition.py @@ -0,0 +1,115 @@ +from datetime import datetime +import os + +from fastapi.encoders import jsonable_encoder +from app import models, schemas +import csv +from app.database.session import SessionLocal +from app.database.seeders.helper import parseFloat, parseInt, calculate_hardware_burden + + +def check_none(value): + if isinstance(value, int) or isinstance(value, float): + return value + return 0 + + +def get_date(year): + if parseInt(year): + return datetime(year=int(year), month=6, day=15) + return None + + + + +def seed() -> None: + db = SessionLocal() + + task = models.Task(name='Named Entity Recognition', + identifier='named-entity-recognition', + description='Named entity recognition (NER) is the task of tagging entities in text with their corresponding type. Approaches typically use BIO notation, which differentiates the beginning (B) and the inside (I) of entities. O is used for non-entity tokens.', + image='https://computerprogress.xyz/image/named-entity-recognition.svg') + + dataset = models.Dataset(name='Conll 2003', identifier='conll-2003') + f1_score = models.AccuracyType(name='F1') + + task_dataset_accuracy_type_f1_score = models.TaskDatasetAccuracyType( + required=True, main=True, accuracy_type=f1_score) + + task_dataset = models.TaskDataset(task=task, dataset=dataset) + + task_dataset.accuracy_types.append(task_dataset_accuracy_type_f1_score) + if os.path.exists('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Named_Entity_Recognition_CoNLL.csv'): + with open('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Named_Entity_Recognition_CoNLL.csv', mode='r') as csv_file: + csv_reader = csv.DictReader(csv_file) + papers = {} + for row in csv_reader: + if papers.get(row.get('title')): + papers[row.get('title')].append(row) + else: + papers[row.get('title')] = [row] + + for _, p in papers.items(): + paper = { + 'title': p[0].get('title'), + 'link': p[0].get('paper_url'), + 'pwc_link': p[0].get('pwc_url'), + 'code_link': p[0].get('code_link'), + 'publication_date': get_date(p[0].get('year')), + 'authors': p[0].get('authors'), + } + paper = models.Paper(**paper) + paper.submission = models.Submission(status='approved') + + count = 0 + for m in p: + count += 1 + + model = { + 'name': m.get('method').strip() if m.get('method') else None, + 'training_time': parseInt(m.get('time_sec')), + 'gflops': + (parseFloat(m.get('flops')) / 10e9) if ( + parseFloat(m.get('flops'))) else None, + 'epochs': parseInt(m.get('epochs')), + 'number_of_parameters': parseInt(m.get('number_of_parameters')), + 'multiply_adds': + (parseFloat(m.get('multiply_adds')) / 10e9) if ( + parseFloat(m.get('multiply_adds'))) else None, + 'number_of_cpus': parseInt(m.get('#cpu')), + 'number_of_gpus': parseInt(m.get('#gpu')), + 'number_of_tpus': parseInt(m.get('#tpu')), + } + try: + + model = jsonable_encoder(model) + schemas.Model(**model) + except Exception: + + continue + model = models.Model(**model) + + model.paper = paper + + if m.get('cpu'): + model.cpu = db.query(models.Cpu).filter( + models.Cpu.name.ilike(m.get('cpu'))).first() + if m.get('gpu'): + model.gpu = db.query(models.Gpu).filter( + models.Gpu.name.ilike(m.get('gpu'))).first() + if m.get('tpu'): + model.tpu = db.query(models.Tpu).filter( + models.Tpu.name.ilike(m.get('tpu'))).first() + + hardware_burden = calculate_hardware_burden(model) + model.hardware_burden = hardware_burden if hardware_burden != 0 else None + + models.AccuracyValue( + value=parseFloat(m.get('F1')), + accuracy_type=f1_score, + model=model + ) + + task_dataset.models.append(model) + db.add(task_dataset) + db.commit() diff --git a/app/database/seeders/object_detection.py b/app/database/seeders/object_detection.py new file mode 100644 index 0000000..6e9af36 --- /dev/null +++ b/app/database/seeders/object_detection.py @@ -0,0 +1,160 @@ +from datetime import datetime +import os + +from fastapi.encoders import jsonable_encoder +from app import models, schemas +import csv +from app.database.session import SessionLocal +from app.database.seeders.helper import parseFloat, parseInt, calculate_hardware_burden + + +def check_none(value): + if isinstance(value, int) or isinstance(value, float): + return value + return 0 + + +def get_date(year): + if parseInt(year): + return datetime(year=int(year), month=6, day=15) + return None + + + +def seed() -> None: + db = SessionLocal() + + task = models.Task(name='Object Detection', identifier='object-detection', + description='Object detection is the task of detecting instances of objects of a certain class within an image. The state-of-the-art methods can be categorized into two main types: one-stage methods and two stage-methods. One-stage methods prioritize inference speed, and example models include YOLO, SSD and RetinaNet. Two-stage methods prioritize detection accuracy, and example models include Faster R-CNN, Mask R-CNN and Cascade R-CNN. The most popular benchmark is the MSCOCO dataset. Models are typically evaluated according to a Mean Average Precision metric.', + image='https://computerprogress.xyz/image/object-detection.svg' + ) + + dataset = models.Dataset(name='MS COCO', identifier='ms-coco') + + box_ap = models.AccuracyType(name='BOX AP') + ap50 = models.AccuracyType(name='AP50') + ap75 = models.AccuracyType(name='AP75') + aps = models.AccuracyType(name='APS') + apm = models.AccuracyType(name='APM') + apl = models.AccuracyType(name='APL') + + task_dataset_accuracy_type_box_ap = models.TaskDatasetAccuracyType( + required=True, main=True, accuracy_type=box_ap) + task_dataset_accuracy_type_ap50 = models.TaskDatasetAccuracyType( + required=False, main=False, accuracy_type=ap50) + task_dataset_accuracy_type_ap75 = models.TaskDatasetAccuracyType( + required=False, main=False, accuracy_type=ap75) + task_dataset_accuracy_type_aps = models.TaskDatasetAccuracyType( + required=False, main=False, accuracy_type=aps) + task_dataset_accuracy_type_apm = models.TaskDatasetAccuracyType( + required=False, main=False, accuracy_type=apm) + task_dataset_accuracy_type_apl = models.TaskDatasetAccuracyType( + required=False, main=False, accuracy_type=apl) + + task_dataset = models.TaskDataset(task=task, dataset=dataset) + + task_dataset.accuracy_types.append(task_dataset_accuracy_type_box_ap) + task_dataset.accuracy_types.append(task_dataset_accuracy_type_ap50) + task_dataset.accuracy_types.append(task_dataset_accuracy_type_ap75) + task_dataset.accuracy_types.append(task_dataset_accuracy_type_aps) + task_dataset.accuracy_types.append(task_dataset_accuracy_type_apm) + task_dataset.accuracy_types.append(task_dataset_accuracy_type_apl) + + if os.path.exists('app/database/seeders/data/PaperWithCode_Integration_Data_CV_Object_Detection_MS_COCO.csv'): + with open('app/database/seeders/data/PaperWithCode_Integration_Data_CV_Object_Detection_MS_COCO.csv', mode='r') as csv_file: + csv_reader = csv.DictReader(csv_file) + papers = {} + for row in csv_reader: + if papers.get(row.get('title')): + papers[row.get('title')].append(row) + else: + papers[row.get('title')] = [row] + + for _, p in papers.items(): + paper = { + 'title': p[0].get('title'), + 'link': p[0].get('paper_url'), + 'pwc_link': p[0].get('pwc_url'), + 'code_link': p[0].get('code_link'), + 'publication_date': get_date(p[0].get('year')), + 'authors': p[0].get('authors'), + } + paper = models.Paper(**paper) + paper.submission = models.Submission(status='approved') + + count = 0 + for m in p: + count += 1 + + model = { + 'name': m.get('method').strip() if m.get('method') else None, + 'training_time': parseInt(m.get('time_sec')), + 'gflops': + (parseFloat(m.get('flops')) / 10e9) if ( + parseFloat(m.get('flops'))) else None, + 'epochs': parseInt(m.get('#epochs')), + 'number_of_parameters': parseInt(m.get('#params')), + 'multiply_adds': + (parseFloat(m.get('multiadds')) / 10e9) if ( + parseFloat(m.get('multiadds'))) else None, + 'number_of_cpus': parseInt(m.get('#cpu')), + 'number_of_gpus': parseInt(m.get('#gpu')), + 'number_of_tpus': parseInt(m.get('#tpu')), + } + try: + + model = jsonable_encoder(model) + schemas.Model(**model) + except Exception: + + continue + model = models.Model(**model) + + model.paper = paper + + if m.get('cpu'): + model.cpu = db.query(models.Cpu).filter( + models.Cpu.name.ilike(m.get('cpu'))).first() + if m.get('gpu'): + model.gpu = db.query(models.Gpu).filter( + models.Gpu.name.ilike(m.get('gpu'))).first() + if m.get('tpu'): + model.tpu = db.query(models.Tpu).filter( + models.Tpu.name.ilike(m.get('tpu'))).first() + + hardware_burden = calculate_hardware_burden(model) + model.hardware_burden = hardware_burden if hardware_burden != 0 else None + + models.AccuracyValue( + value=parseFloat(m.get('BOX_AP')), + accuracy_type=box_ap, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('AP50')), + accuracy_type=ap50, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('AP75')), + accuracy_type=ap75, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('APS')), + accuracy_type=aps, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('APM')), + accuracy_type=apm, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('APL')), + accuracy_type=apl, + model=model + ) + task_dataset.models.append(model) + db.add(task_dataset) + db.commit() diff --git a/app/database/seeders/question_answering.py b/app/database/seeders/question_answering.py new file mode 100644 index 0000000..a521d0f --- /dev/null +++ b/app/database/seeders/question_answering.py @@ -0,0 +1,121 @@ +from datetime import datetime +import os + +from fastapi.encoders import jsonable_encoder +from app import models, schemas +import csv +from app.database.session import SessionLocal +from app.database.seeders.helper import parseFloat, parseInt, calculate_hardware_burden + + +def check_none(value): + if isinstance(value, int) or isinstance(value, float): + return value + return 0 + + +def get_date(year): + if parseInt(year): + return datetime(year=int(year), month=6, day=15) + return None + + +def seed() -> None: + db = SessionLocal() + + task = models.Task(name='Question Answering', identifier='question-answering', + description='Question Answering is the task of answering questions (typically reading comprehension questions), but abstaining when presented with a question that cannot be answered based on the provided context.', + image='https://computerprogress.xyz/image/question-answering.svg') + + dataset = models.Dataset(name='SQuAD 1.1', identifier='squad11') + + F1 = models.AccuracyType(name='F1') + EM = models.AccuracyType(name='EM') + + task_dataset_accuracy_type_F1 = models.TaskDatasetAccuracyType( + required=True, main=True, accuracy_type=F1) + task_dataset_accuracy_type_EM = models.TaskDatasetAccuracyType( + required=False, main=False, accuracy_type=EM) + + task_dataset = models.TaskDataset(task=task, dataset=dataset) + + task_dataset.accuracy_types.append(task_dataset_accuracy_type_F1) + task_dataset.accuracy_types.append(task_dataset_accuracy_type_EM) + if os.path.exists('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Question_Answering_SQuAD_1_1.csv'): + with open('app/database/seeders/data/PaperWithCode_Integration_Data_NLP_Question_Answering_SQuAD_1_1.csv', mode='r') as csv_file: + csv_reader = csv.DictReader(csv_file) + papers = {} + for row in csv_reader: + if papers.get(row.get('title')): + papers[row.get('title')].append(row) + else: + papers[row.get('title')] = [row] + + for _, p in papers.items(): + paper = { + 'title': p[0].get('title'), + 'link': p[0].get('paper_url'), + 'pwc_link': p[0].get('pwc_url'), + 'code_link': p[0].get('code_link'), + 'publication_date': get_date(p[0].get('year')), + 'authors': p[0].get('authors'), + } + paper = models.Paper(**paper) + paper.submission = models.Submission(status='approved') + + count = 0 + for m in p: + count += 1 + + model = { + 'name': m.get('method').strip() if m.get('method') else None, + 'training_time': parseInt(m.get('time_sec')), + 'gflops': + (parseFloat(m.get('flops')) / 10e9) if ( + parseFloat(m.get('flops'))) else None, + 'epochs': parseInt(m.get('#epochs')), + 'number_of_parameters': parseInt(m.get('#params')), + 'multiply_adds': + (parseFloat(m.get('multiadds')) / 10e9) if ( + parseFloat(m.get('multiadds'))) else None, + 'number_of_cpus': parseInt(m.get('#cpu')), + 'number_of_gpus': parseInt(m.get('#gpu')), + 'number_of_tpus': parseInt(m.get('#tpu')), + } + try: + + model = jsonable_encoder(model) + schemas.Model(**model) + except Exception: + + continue + model = models.Model(**model) + + model.paper = paper + + if m.get('cpu'): + model.cpu = db.query(models.Cpu).filter( + models.Cpu.name.ilike(m.get('cpu'))).first() + if m.get('gpu'): + model.gpu = db.query(models.Gpu).filter( + models.Gpu.name.ilike(m.get('gpu'))).first() + if m.get('tpu'): + model.tpu = db.query(models.Tpu).filter( + models.Tpu.name.ilike(m.get('tpu'))).first() + + hardware_burden = calculate_hardware_burden(model) + model.hardware_burden = hardware_burden if hardware_burden != 0 else None + + models.AccuracyValue( + value=parseFloat(m.get('F1')), + accuracy_type=F1, + model=model + ) + models.AccuracyValue( + value=parseFloat(m.get('EM')), + accuracy_type=EM, + model=model + ) + task_dataset.models.append(model) + db.add(task_dataset) + db.commit() diff --git a/app/models/paper.py b/app/models/paper.py index 7f9e5dd..ad103ff 100644 --- a/app/models/paper.py +++ b/app/models/paper.py @@ -18,6 +18,10 @@ class Paper(Base): default=generate_identifier, onupdate=generate_identifier) title = Column(String) link = Column(String) + + pwc_link = Column(String) + + code_link = Column(String) publication_date = Column(Date) authors = Column(ARRAY(String)) diff --git a/app/routes/login.py b/app/routes/login.py index 5591010..b4326ff 100644 --- a/app/routes/login.py +++ b/app/routes/login.py @@ -93,3 +93,26 @@ def reset_password( db.add(user) db.commit() return {"msg": "Password updated successfully"} + + +@router.post("/confirm-email/", response_model=schemas.Msg) +def confirm_email( + token: str = Body(..., embed=True), + db: Session = Depends(deps.get_db), +) -> Any: + """ + Reset password + """ + email = verify_password_reset_token(token) + if not email: + raise HTTPException(status_code=400, detail="Invalid token") + user = crud.user.get_by_email(db, email=email) + if not user: + raise HTTPException( + status_code=404, + detail="The user with this username does not exist in the system.", + ) + user.is_active = True + db.add(user) + db.commit() + return {"msg": "email confirmed successfully"} diff --git a/app/routes/paper_with_code_integration.py b/app/routes/paper_with_code_integration.py index 9634f06..fa64241 100644 --- a/app/routes/paper_with_code_integration.py +++ b/app/routes/paper_with_code_integration.py @@ -1,6 +1,6 @@ from typing import Any, List -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends from sqlalchemy.orm import Session from app import crud, deps @@ -9,33 +9,18 @@ @router.get("/{task_dataset_identifier}", response_model=List[Any]) -def get_home_info( +def get_models_metrics( db: Session = Depends(deps.get_db), skip: int = 0, limit: int = 100, task_dataset_identifier: str = None, + model_name: str = None, + paper_title: str = None ) -> List[Any]: """ Retrieve tasks. """ models = crud.paper_with_code.get_multi_model_metrics_by_identifier( - db, skip=skip, limit=limit, task_dataset_identifier=task_dataset_identifier) + db, skip=skip, limit=limit, task_dataset_identifier=task_dataset_identifier, + model_name=model_name, paper_title=paper_title) return models - - -@router.get("/{task_dataset_identifier}/{model_identifier}", response_model=Any) -def get_all_datasets_for_task( - *, - db: Session = Depends(deps.get_db), - task_dataset_identifier: str = None, - model_identifier: str = None, -) -> Any: - """ - Get task by ID. - """ - model = crud.paper_with_code.get_model_metrics_by_identifier( - db=db, task_dataset_identifier=task_dataset_identifier, - model_identifier=model_identifier) - if not model: - raise HTTPException(status_code=404, detail="Model not found") - return model diff --git a/app/routes/submission.py b/app/routes/submission.py index 2e0b10c..fc6d43b 100644 --- a/app/routes/submission.py +++ b/app/routes/submission.py @@ -1,7 +1,9 @@ +from pydantic.main import BaseModel +from starlette.background import BackgroundTasks from app.models.submission import StatusEnum from typing import Any, List -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks from fastapi.param_functions import Body from sqlalchemy.orm import Session @@ -10,13 +12,19 @@ router = APIRouter() -@router.get("/", response_model=List[schemas.Submission]) +class SubmissionsWithTotal(BaseModel): + total: int + items: List[schemas.Submission] + + +@router.get("/", response_model=SubmissionsWithTotal) def read_submissions( db: Session = Depends(deps.get_db), skip: int = 0, limit: int = 100, q: str = None, owner_id: int = None, + status: StatusEnum = None, current_user: models.User = Depends(deps.GetCurrentUser('default')), ) -> Any: """ @@ -24,11 +32,11 @@ def read_submissions( """ if current_user.role.value == 'default': submission = crud.submission.get_multi( - db, skip=skip, limit=limit, owner_id=current_user.id, q=q + db, skip=skip, limit=limit, owner_id=current_user.id, q=q, status=status ) else: submission = crud.submission.get_multi( - db, skip=skip, limit=limit, owner_id=owner_id, q=q) + db, skip=skip, limit=limit, owner_id=owner_id, q=q, status=status) return submission @@ -53,6 +61,7 @@ def create_submission( @router.put("/{id}/status", response_model=schemas.Submission) def update_status_submission( *, + background_tasks: BackgroundTasks, db: Session = Depends(deps.get_db), id: int, status: StatusEnum = Body(..., embed=True), @@ -66,7 +75,8 @@ def update_status_submission( raise HTTPException(status_code=404, detail="Submission not found") print(status, flush=True) submission = crud.submission.update_status( - db=db, db_obj=submission, status=status, current_user=current_user) + db=db, db_obj=submission, status=status, current_user=current_user, + background_tasks=background_tasks) return submission diff --git a/app/routes/user.py b/app/routes/user.py index 6d64db1..d7cf220 100644 --- a/app/routes/user.py +++ b/app/routes/user.py @@ -1,6 +1,7 @@ +from app.utils.security import verify_password from typing import Any, List -from fastapi import APIRouter, Body, Depends, HTTPException +from fastapi import APIRouter, Body, Depends, HTTPException, BackgroundTasks from fastapi.encoders import jsonable_encoder from pydantic.networks import EmailStr from sqlalchemy.orm import Session @@ -19,12 +20,16 @@ def read_users( skip: int = 0, limit: int = 100, current_user: models.User = Depends(deps.GetCurrentUser('admin')), + email: str = None ) -> Any: """ Retrieve users. """ - users = crud.user.get_multi(db, skip=skip, limit=limit) - return users + if email: + user = crud.user.get_by_email(db, email=email) + return [user] if user else [] + else: + return crud.user.get_multi(db, skip=skip, limit=limit) @router.post("/", response_model=schemas.User) @@ -43,11 +48,7 @@ def create_user( status_code=400, detail="The user with this username already exists in the system.", ) - user = crud.user.create(db, obj_in=user_in) - if settings.EMAILS_ENABLED and user_in.email: - send_new_account_email( - email_to=user_in.email, username=user_in.email, password=user_in.password - ) + user = crud.user.create(db, obj_in=user_in, is_active=True) return user @@ -55,7 +56,8 @@ def create_user( def update_user_me( *, db: Session = Depends(deps.get_db), - password: str = Body(None), + password: str = Body(None, min_length=8), + current_password: str = Body(None), first_name: str = Body(None), last_name: str = Body(None), email: EmailStr = Body(None), @@ -67,6 +69,14 @@ def update_user_me( current_user_data = jsonable_encoder(current_user) user_in = schemas.UserUpdate(**current_user_data) if password is not None: + if current_password is None: + raise HTTPException( + status_code=400, detail="Current password is missing" + ) + if not verify_password(current_password, current_user.hashed_password): + raise HTTPException( + status_code=400, detail="Incorrect current password" + ) user_in.password = password if first_name is not None: user_in.first_name = first_name @@ -92,8 +102,9 @@ def read_user_me( @router.post("/open", response_model=schemas.User) def create_user_open( *, + background_tasks: BackgroundTasks, db: Session = Depends(deps.get_db), - password: str = Body(...), + password: str = Body(..., min_length=8), email: EmailStr = Body(...), first_name: str = Body(None), last_name: str = Body(None), @@ -113,8 +124,13 @@ def create_user_open( detail="The user with this email already exists in the system", ) user_in = schemas.UserCreate( - password=password, email=email, first_name=first_name, last_name=last_name) + password=password, email=email, first_name=first_name, last_name=last_name, + is_active=False) user = crud.user.create(db, obj_in=user_in) + if settings.EMAILS_ENABLED and user.email: + background_tasks.add_task(send_new_account_email, + email_to=user.email, first_name=user.first_name) + return user diff --git a/app/routes/util.py b/app/routes/util.py index 60055e4..93afc3c 100644 --- a/app/routes/util.py +++ b/app/routes/util.py @@ -18,6 +18,7 @@ def test_email( """ Test emails. """ + print(email_to, flush=True) background_tasks.add_task(send_test_email, email_to=email_to) return {"msg": "Test email sent"} diff --git a/app/schemas/message.py b/app/schemas/message.py index f304ea1..31f854d 100644 --- a/app/schemas/message.py +++ b/app/schemas/message.py @@ -1,3 +1,4 @@ +from datetime import datetime from app.schemas.user import User from typing import Optional @@ -21,6 +22,7 @@ class MessageInDBBase(MessageBase): submission_id: Optional[int] author: Optional[User] type: Optional[str] + created_at: datetime class Config: orm_mode = True diff --git a/app/schemas/paper.py b/app/schemas/paper.py index 7a14910..31708fd 100644 --- a/app/schemas/paper.py +++ b/app/schemas/paper.py @@ -32,7 +32,6 @@ class PaperModelsCreate(BaseModel): number_of_parameters: Optional[int] training_time: Optional[int] epochs: Optional[int] - extra_training_data: bool = False accuracies: List[AccuracyValuesCreate] diff --git a/app/schemas/submission.py b/app/schemas/submission.py index bce9416..e82bbd0 100644 --- a/app/schemas/submission.py +++ b/app/schemas/submission.py @@ -1,3 +1,4 @@ +from app.schemas.user import User from app.models.submission import StatusEnum from typing import List, Optional from datetime import date, datetime @@ -22,7 +23,6 @@ class SubmissionDataModels(BaseModel): number_of_parameters: Optional[int] training_time: Optional[int] epochs: Optional[int] - extra_training_data: bool = False accuracies: List[SubmissionDataAccuracyValues] number_of_gpus: int number_of_cpus: Optional[int] @@ -40,9 +40,10 @@ class SubmissionData(BaseModel): class SubmissionBase(BaseModel): - data: SubmissionData + data: Optional[SubmissionData] paper_id: Optional[int] owner_id: Optional[int] + owner: Optional[User] reviewer_id: Optional[int] status: StatusEnum diff --git a/app/schemas/user.py b/app/schemas/user.py index 486d759..ebda0ae 100644 --- a/app/schemas/user.py +++ b/app/schemas/user.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import BaseModel, EmailStr +from pydantic import BaseModel, EmailStr, constr from app.models.user import RoleEnum @@ -17,12 +17,12 @@ class UserBase(BaseModel): # Properties to receive via API on creation class UserCreate(UserBase): email: EmailStr - password: str + password: constr(min_length=8) # Properties to receive via API on update class UserUpdate(UserBase): - password: Optional[str] = None + password: Optional[constr(min_length=8)] = None class UserInDBBase(UserBase): diff --git a/app/utils/email-templates/email_confirmation.html b/app/utils/email-templates/email_confirmation.html new file mode 100644 index 0000000..1a3a5b2 --- /dev/null +++ b/app/utils/email-templates/email_confirmation.html @@ -0,0 +1,22 @@ +

{{first_name}}, welcome to {{ project_name }}
Confirm email
or click on the link below:
{{ link }}

\ No newline at end of file diff --git a/app/utils/email-templates/email_confirmation.mjml b/app/utils/email-templates/email_confirmation.mjml new file mode 100644 index 0000000..5a2d013 --- /dev/null +++ b/app/utils/email-templates/email_confirmation.mjml @@ -0,0 +1,14 @@ + + + + + + {{first_name}}, welcome to {{ project_name }} + Confirm email + or click on the link below: + {{ link }} + + + + + \ No newline at end of file diff --git a/app/utils/email-templates/new_account.html b/app/utils/email-templates/new_account.html deleted file mode 100644 index 1a57e0b..0000000 --- a/app/utils/email-templates/new_account.html +++ /dev/null @@ -1,26 +0,0 @@ -

{{ project_name }} - New Account
You have a new account:
Username: {{ username }}
Password: {{ password }}
Go to Dashboard

\ No newline at end of file diff --git a/app/utils/email-templates/new_account.mjml b/app/utils/email-templates/new_account.mjml deleted file mode 100644 index a9affd1..0000000 --- a/app/utils/email-templates/new_account.mjml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - {{ project_name }} - New Account - You have a new account: - Username: {{ username }} - Password: {{ password }} - Go to Dashboard - - - - - \ No newline at end of file diff --git a/app/utils/email-templates/submission_updates.html b/app/utils/email-templates/submission_updates.html new file mode 100644 index 0000000..d4d3d69 --- /dev/null +++ b/app/utils/email-templates/submission_updates.html @@ -0,0 +1,22 @@ +

Computer Progress
{{ message }}
Please, sign in to your account and keep contributing to the Computer Progress!

\ No newline at end of file diff --git a/app/utils/email-templates/submission_updates.mjml b/app/utils/email-templates/submission_updates.mjml new file mode 100644 index 0000000..c72fbc9 --- /dev/null +++ b/app/utils/email-templates/submission_updates.mjml @@ -0,0 +1,13 @@ + + + + + + Computer Progress + {{ message }} + Please, sign in to your account and keep contributing to the Computer Progress! + + + + + diff --git a/app/utils/email.py b/app/utils/email.py index daf0ee4..b235511 100644 --- a/app/utils/email.py +++ b/app/utils/email.py @@ -31,6 +31,7 @@ def send_email( smtp_options["password"] = settings.SMTP_PASSWORD response = message.send(to=email_to, render=environment, smtp=smtp_options) logging.info(f"send email result: {response}") + print(f"send email result: {response}", flush=True) def send_test_email(email_to: str) -> None: @@ -67,26 +68,41 @@ def send_reset_password_email(email_to: str, email: str, token: str) -> None: ) -def send_new_account_email(email_to: str, username: str, password: str) -> None: +def send_new_account_email(email_to: str, first_name: str) -> None: project_name = settings.PROJECT_NAME - subject = f"{project_name} - New account for user {username}" - with open(Path(settings.EMAIL_TEMPLATES_DIR) / "new_account.html") as f: + subject = f"{project_name} - Email confirmation" + with open(Path(settings.EMAIL_TEMPLATES_DIR) / "email_confirmation.html") as f: template_str = f.read() - link = settings.SERVER_HOST + link = settings.SERVER_HOST + '/signin?confirmation=' + \ + generate_email_confirmation_token(email=email_to) send_email( email_to=email_to, subject_template=subject, html_template=template_str, environment={ "project_name": settings.PROJECT_NAME, - "username": username, - "password": password, + "first_name": first_name, "email": email_to, "link": link, }, ) +def send_submission_updates_email(email_to: str, message: str) -> None: + project_name = settings.PROJECT_NAME + subject = f"{project_name} - submission update" + with open(Path(settings.EMAIL_TEMPLATES_DIR) / "submission_updates.html") as f: + template_str = f.read() + send_email( + email_to=email_to, + subject_template=subject, + html_template=template_str, + environment={ + "message": message, + }, + ) + + def generate_password_reset_token(email: str) -> str: delta = timedelta(hours=settings.EMAIL_RESET_TOKEN_EXPIRE_HOURS) now = datetime.utcnow() @@ -98,9 +114,16 @@ def generate_password_reset_token(email: str) -> str: return encoded_jwt +def generate_email_confirmation_token(email: str) -> str: + encoded_jwt = jwt.encode( + {"sub": email}, settings.SECRET_KEY, algorithm="HS256", + ) + return encoded_jwt + + def verify_password_reset_token(token: str) -> Optional[str]: try: decoded_token = jwt.decode(token, settings.SECRET_KEY, algorithms=["HS256"]) - return decoded_token["email"] + return decoded_token["sub"] except jwt.JWTError: return None