diff --git a/.github/workflows/check_makefiles.yml b/.github/workflows/check_makefiles.yml index 9ffef3b..bf7ecab 100644 --- a/.github/workflows/check_makefiles.yml +++ b/.github/workflows/check_makefiles.yml @@ -25,9 +25,9 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Runs a single command using the runners shell - name: Check Makefiles - working-directory: tasks/check_makefiles/code + working-directory: ./tasks/actions_scripts/code run: bash check_makefiles.sh ${{ github.event.inputs.tasks }} diff --git a/.github/workflows/check_nontask_makefiles.yml b/.github/workflows/check_nontask_makefiles.yml index b71efdd..0a9d10e 100644 --- a/.github/workflows/check_nontask_makefiles.yml +++ b/.github/workflows/check_nontask_makefiles.yml @@ -21,9 +21,9 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # Runs a single command using the runners shell - name: Check Makefiles - working-directory: tasks/check_makefiles/code + working-directory: ./tasks/actions_scripts/code run: bash check_nontask_makefiles.sh \ No newline at end of file diff --git a/.github/workflows/check_tex_files.yml b/.github/workflows/check_tex_files.yml new file mode 100644 index 0000000..775d849 --- /dev/null +++ b/.github/workflows/check_tex_files.yml @@ -0,0 +1,66 @@ +name: Compile slides and paper + +on: + pull_request: + types: [opened, reopened, review_requested, ready_for_review] + workflow_dispatch: + +jobs: + build_latex: + runs-on: ubuntu-latest + + steps: + # Step to check out the repository + - uses: actions/checkout@v4 + + # Make inputs for paper + - name: Make inputs for paper + run: | + cd paper + make inputs + + # Make inputs for slides + - name: Make inputs for slides + run: | + cd slides + make inputs + + # Step to compile the paper + - name: Compile paper + shell: bash + run: | + bash "./tasks/actions_scripts/code/check_tex_files.sh" + env: + INPUT_ROOT_FILE: paper.tex + INPUT_WORKING_DIRECTORY: ./paper/ + INPUT_WORK_IN_ROOT_FILE_DIR: "false" + INPUT_CONTINUE_ON_ERROR: "true" + INPUT_COMPILER: latexmk + INPUT_ARGS: "-pdf -file-line-error -halt-on-error -interaction=nonstopmode" + INPUT_EXTRA_SYSTEM_PACKAGES: "" + INPUT_EXTRA_FONTS: "" + INPUT_PRE_COMPILE: "" + INPUT_POST_COMPILE: "" + INPUT_LATEXMK_SHELL_ESCAPE: "true" + INPUT_LATEXMK_USE_LUALATEX: "false" + INPUT_LATEXMK_USE_XELATEX: "false" + + # Step to compile the slides + - name: Compile slides + shell: bash + run: | + bash "./tasks/actions_scripts/code/check_tex_files.sh" + env: + INPUT_ROOT_FILE: slides.tex + INPUT_WORKING_DIRECTORY: ./slides/ + INPUT_WORK_IN_ROOT_FILE_DIR: "false" + INPUT_CONTINUE_ON_ERROR: "true" + INPUT_COMPILER: latexmk + INPUT_ARGS: "-pdf -file-line-error -halt-on-error -interaction=nonstopmode" + INPUT_EXTRA_SYSTEM_PACKAGES: "" + INPUT_EXTRA_FONTS: "" + INPUT_PRE_COMPILE: "" + INPUT_POST_COMPILE: "" + INPUT_LATEXMK_SHELL_ESCAPE: "true" + INPUT_LATEXMK_USE_LUALATEX: "false" + INPUT_LATEXMK_USE_XELATEX: "false" diff --git a/bib/aeanobold-oxford.bst b/bib/aeanobold-oxford.bst new file mode 100644 index 0000000..aaf8fcb --- /dev/null +++ b/bib/aeanobold-oxford.bst @@ -0,0 +1,1279 @@ +% BibTeX standard bibliography style `aea' (one of the harvard family) + % version 0.99a for BibTeX versions 0.99a or later, LaTeX version 2.09. + % Copyright (C) 2008, all rights reserved. + % Melissa Rice (melissa@fmtek.net), Full Moon Technical Solutions, LLC (dba FMTek) + % last revised: 20 May 2009 + +% adding Oxford comma: https://tex.stackexchange.com/questions/215403/revise-the-aer-bibliography-style-to-write-out-all-names-if-a-piece-has-three-au + +ENTRY + { address + author + booktitle + chapter + edition + editor + howpublished + institution + journal + key + month + note + number + organization + pages + presented + publisher + school + series + title + type + URL + volume + year + } + { field.used etal.allowed etal.required} + { extra.label sort.label list.year } + +STRINGS { s t f } + +% BOOLEAN OPERATORS + +FUNCTION {not} +{ { #0 } + { #1 } + if$ +} + +FUNCTION {and} +{ 'skip$ + { pop$ #0 } + if$ +} + +FUNCTION {or} +{ { pop$ #1 } + 'skip$ + if$ +} + +% CHECKS + +FUNCTION {author.or.editor.check} +{ author empty$ editor empty$ and + { "empty author and editor in " cite$ * warning$ } + { skip$ } + if$ +} + +FUNCTION {chapter.or.pages.check} +{ chapter empty$ pages empty$ and + { "empty chapter and pages in " cite$ * warning$ } + { skip$ } + if$ +} + + +FUNCTION {required} +{ 't := % name of variable + empty$ % value of variable on stack + { "empty " t * " in " * cite$ * warning$ } + { skip$ } + if$ +} + +FUNCTION {field.or.null} +{ duplicate$ empty$ + { pop$ "" } + 'skip$ + if$ +} + +% GENERAL FORMATTING + +FUNCTION {emphasize} +{ duplicate$ empty$ + { pop$ "" } + { "{\em " swap$ * "}" * } + if$ +} + +FUNCTION {embolden} +{ duplicate$ empty$ + { pop$ "" } + { "{\bf " swap$ * "}" * } + if$ +} + +FUNCTION {quote} +{ duplicate$ empty$ + { pop$ "" } + { "`" swap$ * "'" * } + if$ +} + +FUNCTION {doublequote} +{ duplicate$ empty$ + { pop$ "" } + { "``" swap$ * "''" * } + if$ +} + +FUNCTION {join} +{ 't := % second string + 'f := % join character + 's := % first string + s empty$ + { t empty$ + { "" } + { t } + if$ + } + { t empty$ + { s } + { f ". " = + { s add.period$ " " * t * } + { s f * t * } + if$ + } + if$ + } + if$ +} + +FUNCTION {tie.or.space.connect} +{ duplicate$ text.length$ #3 < + { "~" } + { " " } + if$ + swap$ * * +} + +INTEGERS { len } + +FUNCTION {chop.word} +{ 's := + 'len := + s #1 len substring$ = + { s len #1 + global.max$ substring$ } + 's + if$ +} + +INTEGERS { ind tsslen } + +STRINGS { tss ret rss istr } + +FUNCTION {replace.substring}{ + 'rss := + 'tss := + 'istr := + "" 'ret := + tss text.length$ 'tsslen := + #1 'ind := + { istr ind tsslen substring$ "" = not } + { istr ind tsslen substring$ tss = + { ret rss * 'ret := + ind tsslen + 'ind := + } + { ret istr ind #1 substring$ * 'ret := + ind #1 + 'ind := + } + if$ + } + while$ + ret +} + +FUNCTION {n.dashify} +{ 't := + "" + { t empty$ not } + { t #1 #1 substring$ "-" = + { t #1 #2 substring$ "--" = not + { "--" * + t #2 global.max$ substring$ 't := + } + { { t #1 #1 substring$ "-" = } + { "-" * + t #2 global.max$ substring$ 't := + } + while$ + } + if$ + } + { t #1 #1 substring$ * + t #2 global.max$ substring$ 't := + } + if$ + } + while$ +} + +INTEGERS { multiresult } + +FUNCTION {multi.page.check} +{ 't := + #0 'multiresult := + { multiresult not + t empty$ not + and + } + { t #1 #1 substring$ + duplicate$ "-" = + swap$ duplicate$ "," = + swap$ "+" = + or or + { #1 'multiresult := } + { t #2 global.max$ substring$ 't := } + if$ + } + while$ + multiresult +} + + +% OUTPUT FUNCTIONS + +FUNCTION {output} % writ +{ duplicate$ empty$ + { pop$ } + { write$} + if$ +} + +FUNCTION {output.year} % writ +{ list.year empty$ + { skip$ } + { list.year ". " * write$} + if$ +} + +FUNCTION {output.title.quoted} +{ title empty$ + { skip$ } + { title add.period$ doublequote " " * write$ } + if$ +} + +FUNCTION {output.title.italic} +{ title empty$ + { skip$ } + { title add.period$ emphasize " " * write$ } + if$ +} + +FUNCTION {output.entry} +{ duplicate$ "" = + { pop$ } + { add.period$ + write$ + } + if$ + newline$ +} + +% FIELD FORMATTING + +FUNCTION {format.address.only} +{ address empty$ + { "" } + { address } + if$ +} + +FUNCTION {format.journal} +{ journal empty$ + { "" } + { journal emphasize } + if$ +} + +FUNCTION {format.type} +{ type empty$ + { "" } + { type } + if$ +} + +FUNCTION {format.number} +{ number empty$ + { "" } + { number } + if$ +} + + +FUNCTION {format.school} +{ school empty$ + { "" } + { school } + if$ +} + +FUNCTION {format.institution} +{ institution empty$ + { "" } + { institution } + if$ +} + +FUNCTION {format.masters} +{ "Master's diss." +} + +FUNCTION {format.phd} +{ "PhD diss." +} + +FUNCTION {format.note} +{ note empty$ + { "" } + { note } + if$ +} + +FUNCTION {format.booktitle} +{ booktitle empty$ + { "" } + { booktitle emphasize } + if$ +} + +FUNCTION {format.booktitle.label} +{ booktitle empty$ + { "" } + { booktitle embolden } + if$ +} + +FUNCTION {format.title.bold} +{ title empty$ + { "" } + { title add.period$ embolden } + if$ +} + +FUNCTION {format.month} +{ month empty$ + { "" } + { month } + if$ +} + +FUNCTION {format.vol.num.pages} % writ +{ volume empty$ + { number empty$ + {""} + { "there's a number but no volume in " cite$ * warning$ ", (" number * ")" *} + if$ + } + { number empty$ + { volume } + { volume "(" * number * ")" *} + if$ + } + if$ + pages empty$ + { skip$ } + { duplicate$ "" = + { pages n.dashify * } + { ":~" * pages n.dashify * } + if$ + } + if$ +} + +FUNCTION {format.edition} % writ +{ edition empty$ + { "" } + { edition " ed." * } + if$ +} + +FUNCTION {format.book.vol.series.ed} % writ +{ volume empty$ + { series empty$ + { "" } + { series emphasize } + if$ + } + { "Vol." volume tie.or.space.connect + series empty$ + 'skip$ + { " of " * series emphasize * } + if$ + } + if$ + edition empty$ + { skip$ } + { ". " * edition * " ed." *} + if$ +} + +FUNCTION {format.address.publisher} % writ +{ address empty$ + { publisher empty$ + { "" } + { publisher } + if$ + } + { publisher empty$ + { address } + { address ":" * publisher *} + if$ + } + if$ +} + +FUNCTION {format.howpublished} +{ howpublished empty$ + { "" } + { howpublished emphasize } + if$ +} + +FUNCTION {format.organization} +{ organization empty$ + { "" } + { organization } + if$ +} + +FUNCTION {format.organization.presented} +{ organization empty$ + { "" } + { presented empty$ + { organization } + { presented " " * organization * } + if$ + } + if$ +} + +FUNCTION {format.pages} +{ pages empty$ + { "" } + { pages multi.page.check + { pages n.dashify } + { pages } + if$ + } + if$ +} + +FUNCTION {format.chapter.pages} +{ chapter empty$ + { pages empty$ + { "" } + { format.pages } + if$ + } + { pages empty$ + { "Chapter " chapter * } + { "Chapter " chapter * ", " * format.pages * } + if$ + } + if$ +} + +INTEGERS { nameptr namesleft numnames } + +% Format names so that first author is "last, first" and subsequent authors are "first last" with commas +% separating authors whenever there are two or more and "and" precedes last author if 2 or more. +FUNCTION {format.names} +{ 's := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { nameptr #1 = + { s nameptr "{vv~}{ll}{, jj}{, ff}" format.name$ 't := } + { s nameptr "{ff }{vv~}{ll}{, jj}" format.name$ 't := } + if$ + nameptr #1 > + { namesleft #1 > + { ", " * t * } + { t "others" = + { ", et~al." * } + { ", and " * t * } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {format.names.forward} +{ 's := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr "{ff }{vv~}{ll}{, jj}" format.name$ 't := + nameptr #1 > + { namesleft #1 > + { ", " * t * } + { t "others" = + { " et~al." * } + { " and " * t * } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +INTEGERS { author.field editor.field organization.field title.field key.field } + +FUNCTION {init.field.constants} +{ #0 'author.field := + #1 'editor.field := + #2 'organization.field := + #3 'title.field := + #4 'key.field := +} + + + +FUNCTION {format.names.label} +{ 's := + #1 'nameptr := + s num.names$ 'numnames := + numnames 'namesleft := + { namesleft #0 > } + { s nameptr "{ff }{vv~}{ll}{, jj}" format.name$ 't := + nameptr #1 > + { namesleft #1 > + { ", " * t * } + { numnames #2 > + { "," * } + 'skip$ + if$ + t "others" = + { " et~al." * } + { " \harvardand\ " * t * } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ +} + +FUNCTION {format.names.label.short} +{ 's := + #1 'nameptr := + s num.names$ 'numnames := + numnames #3 > + { s #1 "{vv~}{ll}" format.name$ " et~al." *} + { numnames 'namesleft := + { namesleft #0 > } + { s nameptr "{vv~}{ll}" format.name$ 't := + nameptr #1 > + { namesleft #1 > + { ", " * t * } + { numnames #2 > + { "," * } + 'skip$ + if$ + t "others" = + { " et~al." * } + { " \harvardand\ " * t * } + if$ + } + if$ + } + 't + if$ + nameptr #1 + 'nameptr := + namesleft #1 - 'namesleft := + } + while$ + } + if$ +} + +FUNCTION {format.authors} +{ author empty$ + { "" } + { author format.names add.period$} + if$ +} + +FUNCTION {format.editors} +{ editor empty$ + { "" } + { editor format.names add.period$} + if$ +} + +FUNCTION {format.editors.primary} +{ editor empty$ + { "" } + { editor format.names ", " * embolden " ed" *} + if$ +} + +FUNCTION {format.editors.secondary} +{ editor empty$ + { "" } + { ", ed. " editor format.names.forward *} + if$ +} + +FUNCTION {make.list.label} % writ +{ author.field field.used = + { format.authors } + { editor.field field.used = + { format.editors.primary } + { organization.field field.used = + { "The " #4 organization chop.word embolden } + { title.field field.used = + { format.title.bold } + { key.field field.used = + { key } + { "Internal error :001 on " cite$ * " label" * warning$ } + if$ + } + if$ + } + if$ + } + if$ + } + if$ +} + +FUNCTION {output.list.label} % writ +{ make.list.label add.period$ " " * write$ +} + +FUNCTION {make.label.short} % writ +{ author.field field.used = + { author format.names.label.short } + { editor.field field.used = + { editor format.names.label.short } + { organization.field field.used = + { "The " #4 organization chop.word} + { title.field field.used = + { format.booktitle.label } + { key.field field.used = + { key } + {"Internal error :001 on " cite$ * " label" * warning$ } + if$ + } + if$ + } + if$ + } + if$ + } + if$ +} + +FUNCTION {make.label.full} % writ +{ author.field field.used = + { author format.names.label } + { editor.field field.used = + { editor format.names.label } + { organization.field field.used = + { "The " #4 organization chop.word} + { title.field field.used = + { format.booktitle.label } + { key.field field.used = + { key } + {"Internal error :001 on " cite$ * " label" * warning$ } + if$ + } + if$ + } + if$ + } + if$ + } + if$ +} + +% bibitem syntax: +% \harvarditem[abbr-citation]{full-citation}{citation-year}{cite-key} +% abbr-citation: names for 2nd and subsequent citations +% full-citation: names for 1st citation + +% Give full name (first name, middle initial, and last name) of author(s) and year of publication in the first citation. +% Give the last name of author and year of publication in parentheses for each subsequent citation. +% If there are four or more authors, refer to the first author, followed by et al. and the year. + +FUNCTION {output.bibitem} % writ +{ newline$ + "\harvarditem[" make.label.short * "]" * + "{" make.label.full * "}" * * + "{" list.year * "}" * * + "{" cite$ * "}" * * write$ + newline$ +} + +FUNCTION {stack.show} +{ duplicate$ "stack top: " swap$ * " in " * cite$ * warning$ + duplicate$ "" = + { "stack top is blank" warning$ } + { "stack top is non-blank" warning$ } + if$ +} +FUNCTION {article} % writ +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.quoted + author "author" required + title "title" required + journal "journal" required + year "year" required + format.journal ", " +% format.month join ", " + format.vol.num.pages join ". " + format.note join + output.entry +} + +FUNCTION {book} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.italic + author.or.editor.check + title "title" required + publisher "publisher" required + year "year" required +% format.month ", " +% format.book.vol.series.ed join ", " + format.book.vol.series.ed ", " + format.address.publisher join ". " + format.note join + output.entry +} + + +FUNCTION {booklet} +{ output.bibitem + output.list.label % write author or surrogate label + year empty$ + { skip$ } + { output.year } + if$ + author empty$ + { skip$ } + { output.title.quoted } + if$ + title "title" required + format.howpublished ", " +% format.month join ", " + format.address.only join ". " + format.note join + output.entry +} + +FUNCTION {inbook} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.quoted + author.or.editor.check + chapter.or.pages.check + title "title" required + publisher "publisher" required + year "year" required + format.booktitle ", " + format.editors.secondary join " " + format.book.vol.series.ed join ", " + format.chapter.pages join ". " + format.address.publisher join ". " + format.note join + output.entry +} + +FUNCTION {incollection} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.quoted + "In " format.booktitle * ". " * output + author "author" required + title "title" required + booktitle "booktitle" required + publisher "publisher" required + year "year" required + format.book.vol.series.ed ", " + format.editors.secondary join ", " + format.chapter.pages join ". " + format.address.publisher join ". " + format.note join + output.entry +} + +FUNCTION {inproceedings} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.quoted + author "author" required + title "title" required + booktitle "booktitle" required + year "year" required + format.book.vol.series.ed ", " + format.pages join ", " + format.organization.presented join ". " + format.address.publisher join ". " + format.note join + output.entry +} + +FUNCTION {conference} { inproceedings } + +FUNCTION {manual} +{ output.bibitem + output.list.label % write author or surrogate label + title "title" required + year empty$ + { skip$ } + { output.year } + if$ + author empty$ + { skip$ } + { output.title.quoted } + if$ + format.edition ", " + format.address.only join ", " + format.organization join ", " +% format.month join ", " + format.note join + output.entry +} + +FUNCTION {mastersthesis} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.quoted + author "author" required + title "title" required + school "school" required + year "year" required + format.masters " " + format.school join ", " + format.address.only join ", " +% format.month join ", " + format.note join + output.entry +} + +FUNCTION {misc} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + author empty$ + { skip$ } + { output.title.quoted } + if$ + format.howpublished ", " +% format.month join ". " + format.note join + output.entry +} + +FUNCTION {phdthesis} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.quoted + author "author" required + title "title" required + school "school" required + year "year" required + format.phd " " + format.school join ", " + format.address.only join ", " +% format.month join ". " + format.note join + output.entry +} + +FUNCTION {proceedings} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + title "title" required + year "year" required + editor empty$ organization empty$ and + { skip$ } + { output.title.quoted } + if$ + editor empty$ + { "" "" } + { format.organization ", " } + if$ + format.address.publisher join ", " + format.note join + output.entry +} + +FUNCTION {techreport} +{ output.bibitem + output.list.label % write author or surrogate label + output.year + output.title.quoted + author "author" required + title "title" required + institution "institution" required + year "year" required + format.institution " " + format.type join " " + format.number join ", " +% format.month join ", " + format.address.publisher join ". " + format.note join + output.entry +} + +FUNCTION {unpublished} +{ output.bibitem + output.list.label % write author or surrogate label + year empty$ + { skip$ } + { output.year } + if$ + output.title.quoted + author "author" required + title "title" required + note "note" required + format.note + output.entry +} + + +FUNCTION {default.type} { misc } + +MACRO {jan} {"January"} + +MACRO {feb} {"February"} + +MACRO {mar} {"March"} + +MACRO {apr} {"April"} + +MACRO {may} {"May"} + +MACRO {jun} {"June"} + +MACRO {jul} {"July"} + +MACRO {aug} {"August"} + +MACRO {sep} {"September"} + +MACRO {oct} {"October"} + +MACRO {nov} {"November"} + +MACRO {dec} {"December"} + +READ + +EXECUTE {init.field.constants} + +FUNCTION {sortify} +{ purify$ + "l" change.case$ +} + +FUNCTION {sortify.names} +{ " \harvardand\ " " " replace.substring + " et~al." " zzz" replace.substring + sortify +} + +FUNCTION {author.key.label} +{ author empty$ + { key empty$ + { title.field 'field.used := } + { key.field 'field.used := } + if$ + } + { author.field 'field.used := } + if$ +} + +FUNCTION {author.editor.key.label} +{ author empty$ + { editor empty$ + { key empty$ + { title.field 'field.used := } + { key.field 'field.used := } + if$ + } + { editor.field 'field.used := } + if$ + } + { author.field 'field.used := } + if$ +} + +FUNCTION {author.key.organization.label} +{ author empty$ + { key empty$ + { organization empty$ + { title.field 'field.used := } + { organization.field 'field.used := } + if$ + } + { key.field 'field.used := } + if$ + } + { author.field 'field.used := } + if$ +} + +FUNCTION {editor.key.organization.label} +{ editor empty$ + { key empty$ + { organization empty$ + { title.field 'field.used := } + { organization.field 'field.used := } + if$ + } + { key.field 'field.used := } + if$ + } + { editor.field 'field.used := } + if$ +} + +FUNCTION {sort.format.title} +{ 't := + "A " #2 + "An " #3 + "The " #4 t chop.word + chop.word + chop.word + sortify + #1 global.max$ substring$ +} + +FUNCTION {calc.label} +{ make.label.short + title.field field.used = + { sort.format.title } + { sortify.names } + if$ + year field.or.null purify$ #-1 #4 substring$ sortify + * + 'sort.label := +} + +FUNCTION {preliminaries} +{ type$ "book" = + type$ "inbook" = + or + 'author.editor.key.label + { type$ "proceedings" = + 'editor.key.organization.label + { type$ "manual" = + 'author.key.organization.label + 'author.key.label + if$ + } + if$ + } + if$ + author.field field.used = + { + author num.names$ #2 > + { #1 } + { #0 } + if$ + 'etal.required := + } + { + editor.field field.used = + { + editor num.names$ #2 > + { #1 } + { #0 } + if$ + } + { #0 } + if$ + 'etal.required := + } + if$ + #1 'etal.allowed := +} + +FUNCTION {first.presort} +{ calc.label + sort.label + title.field field.used = + { skip$ } + { " " + * + make.list.label sortify.names + * + " " + * + title field.or.null + sort.format.title + * + } + if$ + #1 entry.max$ substring$ + 'sort.key$ := +} + +ITERATE {preliminaries} + +ITERATE {first.presort} + +SORT + +STRINGS { last.sort.label next.extra last.full.label} + +INTEGERS { last.extra.num last.etal.allowed} + +FUNCTION {initialize.confusion} +{ #0 int.to.chr$ 'last.sort.label := + #0 int.to.chr$ 'last.full.label := + #1 'last.etal.allowed := +} + +FUNCTION {confusion.pass} +{ last.sort.label sort.label = + { last.etal.allowed + { last.full.label make.label.full sortify.names = + { skip$ } + { #0 'etal.allowed := + #0 'last.etal.allowed := + } + if$ + } + { #0 'etal.allowed := } + if$ + } + { sort.label 'last.sort.label := + make.label.full sortify.names 'last.full.label := + #1 'last.etal.allowed := + } + if$ +} + +EXECUTE {initialize.confusion} + +ITERATE {confusion.pass} + +EXECUTE {initialize.confusion} + +REVERSE {confusion.pass} + +FUNCTION {initialize.last.extra.num} +{ #0 int.to.chr$ 'last.sort.label := + "" 'next.extra := + #0 'last.extra.num := +} + +FUNCTION {forward.pass} +{ last.sort.label sort.label = + { last.extra.num #1 + 'last.extra.num := + last.extra.num int.to.chr$ 'extra.label := + } + { "a" chr.to.int$ 'last.extra.num := + "" 'extra.label := + sort.label 'last.sort.label := + } + if$ +} + +FUNCTION {reverse.pass} +{ next.extra "b" = + { "a" 'extra.label := } + 'skip$ + if$ + year empty$ + { "n.d." extra.label emphasize * 'list.year := } + { year extra.label emphasize * 'list.year := } + if$ + extra.label 'next.extra := +} + +ITERATE {first.presort} + +SORT + +EXECUTE {initialize.last.extra.num} + +ITERATE {forward.pass} + +REVERSE {reverse.pass} + +FUNCTION {second.presort} +{ make.list.label + title.field field.used = + { sort.format.title } + { sortify.names } + if$ + " " + * + list.year field.or.null sortify + * + " " + * + title.field field.used = + { skip$ } + { title field.or.null + sort.format.title + * + } + if$ + #1 entry.max$ substring$ + 'sort.key$ := +} + +ITERATE {second.presort} + +SORT + +FUNCTION {begin.bib} +{ newline$ newline$ "% Bibstyle aea.bst version 2009.05.20" write$ newline$ newline$ + preamble$ empty$ + 'skip$ + { "\harvardpreambledefs{%" write$ newline$ + preamble$ write$ "}" write$ newline$ + "\harvardpreambletext{%" write$ newline$ + preamble$ write$ "}" write$ newline$ } + if$ + "\begin{thebibliography}{xx}" write$ newline$ +} + +EXECUTE {begin.bib} + +ITERATE {call.type$} + +FUNCTION {end.bib} +{ newline$ + "\end{thebibliography}" write$ newline$ +} + +EXECUTE {end.bib} + diff --git a/bib/bib.bib b/bib/bib.bib new file mode 100644 index 0000000..339762d --- /dev/null +++ b/bib/bib.bib @@ -0,0 +1,156 @@ +%% This BibTeX bibliography file was created using BibDesk. +%% https://bibdesk.sourceforge.io/ + +%% Created for Dingel, Jonathan at 2022-11-20 13:07:39 -0600 + + +%% Saved with string encoding Unicode (UTF-8) + + + +@unpublished{CoutureDingelGreenHandbury:2022, + author = {Victor Couture and Jonathan I. Dingel and Allison Green and Jessie Handbury}, + title = {Demographic Differences in Social Exposure to High-Income People}, + year = {2023}} + +@techreport{DingelGottliebLozinskiMourot:2023, + author = {Jonathan I. Dingel and Joshua D. Gottlieb and Maya Lozinski and Pauline Mourot}, + title = {{Market Size and Trade in Medical Services}}, + institution = "National Bureau of Economic Research", + type = "Working Paper", + series = "Working Paper Series", + number = "31030", + year = "2023", + month = "March" +} + +@unpublished{DingelTintelnot:2023, + author = {Jonathan I. Dingel and Felix Tintelnot}, + date-modified = {2022-11-20 12:04:34 -0600}, + month = {September}, + note = {Revision resubmitted to \textit{Econometrica}}, + title = {{Spatial Economics for Granular Settings}}, + type ={}, + year = {2023}} + +@unpublished{DingelMengHsiang:2023, + author = {Dingel, Jonathan I. and Meng, Kyle C. and Hsiang, Solomon M.}, + month = {July}, + note = {Revision requested by \textit{Review of Economics and Statistics}}, + title = {{Spatial Correlation, Trade, and Inequality: Evidence from the Global Climate}}, + type = {}, + year = {2023}, +} + +@article{CoutureDingelGreenHandburyWilliams:2021, + abstract = {Tracking human activity in real time and at fine spatial scale is particularly valuable during episodes such as the COVID-19 pandemic. In this paper, we discuss the suitability of smartphone data for quantifying movement and social contact. These data cover broad sections of the US population and exhibit pre-pandemic patterns similar to conventional survey data. We develop and make publicly available a location exposure index that summarizes county-to-county movements and a device exposure index that quantifies social contact within venues. We also investigate the reliability of smartphone movement data during the pandemic.}, + author = {Couture, Victor and Dingel, Jonathan I. and Green, Allison and Handbury, Jessie and Williams, Kevin R.}, + doi = {10.1016/j.jue.2021.103328}, + journal = {Journal of Urban Economics}, + number = {C}, + title = {{JUE Insight: Measuring movement and social contact with smartphone data: a real-time application to COVID-19}}, + url = {https://ideas.repec.org/a/eee/juecon/v127y2022ics0094119021000103.html}, + volume = {127}, + year = 2022, + bdsk-url-1 = {https://ideas.repec.org/a/eee/juecon/v127y2022ics0094119021000103.html}, + bdsk-url-2 = {https://doi.org/10.1016/j.jue.2021.103328}} + +@article{DingelMiscioDavis:2021, + abstract = {In developed economies, agglomeration is skill-biased: larger cities are skill-abundant and exhibit higher skilled wage premia. This paper characterizes the spatial distributions of skills in Brazil, China, and India. To facilitate comparisons with developed-economy findings, we construct metropolitan areas for each of these economies by aggregating finer geographic units on the basis of contiguous areas of light in nighttime satellite images. Our results validate this procedure. These lights-based metropolitan areas mirror commuting-based definitions in the United States and Brazil. In China and India, which lack commuting-based definitions, lights-based metropolitan populations follow a power law, while administrative units do not. Examining variation in relative quantities and prices of skill across these metropolitan areas, we conclude that agglomeration is also skill-biased in Brazil, China, and India.}, + author = {Dingel, Jonathan I. and Miscio, Antonio and Davis, Donald R.}, + doi = {10.1016/j.jue.2019.05.005}, + journal = {Journal of Urban Economics}, + keywords = {Cities; Metropolitan areas; Satellite images; Skill-biased agglomeration; Zipf's law}, + number = {C}, + title = {{Cities, lights, and skills in developing economies}}, + url = {https://ideas.repec.org/a/eee/juecon/v125y2021ics0094119019300439.html}, + volume = {125}, + year = 2021, + bdsk-url-1 = {https://ideas.repec.org/a/eee/juecon/v125y2021ics0094119019300439.html}, + bdsk-url-2 = {https://doi.org/10.1016/j.jue.2019.05.005}} + +@article{DingelNeiman:2020, + abstract = {Evaluating the economic impact of ``social distancing'' measures taken to arrest the spread of COVID-19 raises a fundamental question about the modern economy: how many jobs can be performed at home? We classify the feasibility of working at home for all occupations and merge this classification with occupational employment counts. We find that 37\% of jobs in the United States can be performed entirely at home, with significant variation across cities and industries. These jobs typically pay more than jobs that cannot be done at home and account for 46\% of all US wages. Applying our occupational classification to 85 other countries reveals that lower-income economies have a lower share of jobs that can be done at home.}, + author = {Dingel, Jonathan I. and Neiman, Brent}, + doi = {10.1016/j.jpubeco.2020.10}, + journal = {Journal of Public Economics}, + keywords = {Remote work; Telecommuting; Work from home}, + number = {C}, + title = {{How many jobs can be done at home?}}, + url = {https://ideas.repec.org/a/eee/pubeco/v189y2020ics0047272720300992.html}, + volume = {189}, + year = 2020, + bdsk-url-1 = {https://ideas.repec.org/a/eee/pubeco/v189y2020ics0047272720300992.html}, + bdsk-url-2 = {https://doi.org/10.1016/j.jpubeco.2020.10}} + +@article{DavisDingel2020, + abstract = {What determines the distributions of skills, occupations, and industries across cities? We develop a theory to jointly address these fundamental questions about the spatial organization of economies. Our model incorporates a system of cities, their internal urban structures, and a high-dimensional theory of factor-driven comparative advantage. It predicts that larger cities will be skill-abundant and specialize in skill-intensive activities according to the monotone likelihood ratio property. We test the model using data on 270 US metropolitan areas, 3 to 9 educational categories, 22 occupations, and 19 industries. The results provide support for our theory's predictions.}, + author = {Donald R. Davis and Jonathan I. Dingel}, + date-modified = {2022-11-20 12:34:12 -0600}, + doi = {https://doi.org/10.1016/j.jinteco.2020.103291}, + issn = {0022-1996}, + journal = {Journal of International Economics}, + keywords = {Agglomeration, Assignment models, Cities, Comparative advantage}, + pages = {103291}, + title = {The Comparative Advantage of Cities}, + url = {http://www.sciencedirect.com/science/article/pii/S0022199620300106}, + year = {2020}, + bdsk-url-1 = {http://www.sciencedirect.com/science/article/pii/S0022199620300106}, + bdsk-url-2 = {https://doi.org/10.1016/j.jinteco.2020.103291}} + +@article{DavisDingelMonrasMorales:2019, + abstract = {We provide measures of ethnic and racial segregation in urban consumption. Using Yelp reviews, we estimate how spatial and social frictions influence restaurant visits within New York City. Transit time plays a first-order role in consumption choices, so consumption segregation partly reflects residential segregation. Social frictions also affect restaurant choices: individuals are less likely to visit venues in neighborhoods demographically different from their own. While spatial and social frictions jointly produce significant levels of consumption segregation, we find that restaurant consumption is only about half as segregated as residences. Consumption segregation owes more to social than spatial frictions.}, + author = {Donald R. Davis and Jonathan I. Dingel and Joan Monras and Eduardo Morales}, + date-modified = {2022-11-20 11:41:02 -0600}, + doi = {10.1086/701680}, + journal = {Journal of Political Economy}, + number = {4}, + pages = {1684-1738}, + title = {{How Segregated Is Urban Consumption?}}, + url = {https://ideas.repec.org/a/ucp/jpolec/doi10.1086-701680.html}, + volume = {127}, + year = 2019, + bdsk-url-1 = {https://ideas.repec.org/a/ucp/jpolec/doi10.1086-701680.html}, + bdsk-url-2 = {https://doi.org/10.1086/701680}} + +@article{DavisDingel:2019, + abstract = {Leading empiricists and theorists of cities have recently argued that the generation and exchange of ideas must play a more central role in the analysis of cities. This paper develops the first system of cities model with costly idea exchange as the agglomeration force. The model replicates a broad set of established facts about the cross section of cities. It provides the first spatial equilibrium theory of why skill premia are higher in larger cities and how variation in these premia emerges from symmetric fundamentals.}, + author = {Donald R. Davis and Jonathan I. Dingel}, + date-modified = {2022-11-20 11:41:02 -0600}, + journal = {American Economic Review}, + month = {January}, + number = {1}, + pages = {153-170}, + title = {{A Spatial Knowledge Economy}}, + url = {https://ideas.repec.org/a/aea/aecrev/v109y2019i1p153-70.html}, + volume = {109}, + year = 2019, + bdsk-url-1 = {https://ideas.repec.org/a/aea/aecrev/v109y2019i1p153-70.html}} + +@article{Dingel:2017, + abstract = {A growing literature suggests that high-income countries export high-quality goods. Two hypotheses may explain such specialization, with different implications for welfare, inequality, and trade policy. Fajgelbaum et al. formalize the Linder hypothesis that home demand determines the pattern of specialization and, therefore, predict that high-income locations export high-quality products. The factor-proportions model also predicts that skill-abundant, high-income locations export skill-intensive, high-quality products. Prior empirical evidence does not separate these explanations. I develop a model that nests both hypotheses and employ microdata on U.S. manufacturing plants' shipments and factor inputs to quantify the two mechanisms' roles in quality specialization across U.S. cities. Home-market demand explains as much of the relationship between income and quality as differences in factor usage.}, + author = {Jonathan I. Dingel}, + date-modified = {2022-11-20 11:41:02 -0600}, + journal = {Review of Economic Studies}, + keywords = {Quality specialization; Product quality; Market access; Home-market effect}, + number = {4}, + pages = {1551-1582}, + title = {{The Determinants of Quality Specialization}}, + url = {https://ideas.repec.org/a/oup/restud/v84y2017i4p1551-1582..html}, + volume = {84}, + year = 2017, + bdsk-url-1 = {https://ideas.repec.org/a/oup/restud/v84y2017i4p1551-1582..html}} + +@incollection{BaldwinDingel:2022, + abstract = {The Covid-19 pandemic has introduced huge numbers of employers and employees to remote work. How many of these newly remote jobs will go overseas? We offer a rough quantification based on two observations: 1) offshore work is trade in services, and 2) the number of telemigrants is the volume of this trade divided by the average wage. Combining these with gravity-model estimates, we can roughly predict the number of new telemigrants that would arise from lower barriers to trade in services. Telemigration seems unlikely to be transformative when it comes to the development paths of most emerging economies. The baseline service trade flows are modest, and the standard gravity model restricts modest changes to have modest impacts. There are no tipping points in structural gravity models. Finally, we propose a simple model of telemigration in which small changes can have large consequences. The key is to assume that latent comparative advantage takes a different shape than typically assumed in quantitative trade models. Given this, small changes in trade costs can generate large and asymmetric increases in the exports of service tasks from low-wage nations.}, + author = {Richard Baldwin and Jonathan I. Dingel}, + booktitle = {Robots and AI: a New Economic Era}, + chapter = {6}, + date-added = {2022-11-20 11:42:01 -0600}, + date-modified = {2022-11-20 11:43:23 -0600}, + editor = {Gene Grossman and Lili Yan Ing}, + pages = {150-179}, + publisher = {Routledge}, + title = {{Telemigration and Development: On the Offshorability of Teleworkable Jobs}}, + year = 2022, + bdsk-url-1 = {https://ideas.repec.org/p/nbr/nberwo/29387.html}} diff --git a/logbook/entries/codereview.tex b/logbook/entries/codereview.tex index 9440956..eaef276 100755 --- a/logbook/entries/codereview.tex +++ b/logbook/entries/codereview.tex @@ -6,7 +6,9 @@ \subsection{Code review process}\label{code_review_process} Given a task, the default process for writing code and submitting it for review is the following: \begin{enumerate} -\item Create a new branch when you create a new task (\href{https://github.com/gslab-econ/ra-manual/wiki/Tasks}{G\&S}: ``The assignee should create a separate git branch for each task that involves code or output stored in the repository.'') +\item Create a new branch when you create a new task +(\href{https://github.com/gslab-econ/ra-manual/wiki/Tasks}{G\&S}: ``The assignee should create a separate git branch for each task that involves code or output stored in the repository.'') +Either (1) create it on the remote repo using the GitHub web interface and then fetch and check it out or (2) create it locally and then push it to the remote repo. \item Commit your work on this new branch. A complete task includes a \texttt{Makefile}. \item In addition to creating a \texttt{code} folder, write a \texttt{readme.md} in the task folder that explains what it does. \item Submit a pull request when your code is ready to be reviewed. @@ -22,7 +24,10 @@ \subsection{Code review process}\label{code_review_process} (Note: in order for the graph to be informative, your branch must be up-to-date with main.) \item Referencing \texttt{task\_flow\_branchdiff.png}, delete all input and output folders in the tasks that were created or - modified on the branch. + modified on the branch.\footnote{An exception to this rule: + it would be costly to delete the output folder of \texttt{download\_data} every time, + as it bundles all downloads. Instead, only delete the outputs that are relevant to the changes + on the branch.} \item Run \texttt{make} in the tasks that were created or modified on the branch. Verify that committed reports and outputs match those generated on the reviewer's machine. \item Review the code. @@ -42,7 +47,7 @@ \subsection{Code review process}\label{code_review_process} If any fail, fix the mistakes and then re-request review to run the checks again. Note that you can also run the checks manually by executing the shell -scripts in \texttt{tasks/check\_makefiles}. +scripts in \texttt{tasks/actions\_scripts}. Similarly, running \texttt{make} in \texttt{tasks/maintenance} flags scripts and \texttt{Makefile}s that violate best practices or our style preferences. diff --git a/logbook/entries/codestylesheet.tex b/logbook/entries/codestylesheet.tex index 65e4920..82e685b 100755 --- a/logbook/entries/codestylesheet.tex +++ b/logbook/entries/codestylesheet.tex @@ -114,10 +114,6 @@ \subsubsection{Package management} Package dependencies should be identified by \texttt{requirements.txt} (Python), \texttt{Manifest.toml} (Julia), and so forth. Stata lacks a built-in package-dependency management tool. -I just write a do file that installs all the packages, but there is no official requirements/manifest file. -My understanding is that R doesn't do package management properly natively. -I think you want something like ``packrat", but I have never used this myself. - These requirements should be committed to the repo so that anyone cloning the repo on a new machine has everything they need defined. This is also \href{https://aeadataeditor.github.io/aea-de-guidance/preparing-for-data-deposit.html}{AEA journal policy}: ``packages/modules/etc. ... provide a setup program to install these (not manual instructions).'' diff --git a/logbook/entries/git_alone.tex b/logbook/entries/git_alone.tex index 9a9dacd..c88e7f0 100644 --- a/logbook/entries/git_alone.tex +++ b/logbook/entries/git_alone.tex @@ -63,7 +63,7 @@ \subsection{Version control in a single-machine environment} \begin{itemize} \item For command-line Git, see Atlassian's \href{https://www.atlassian.com/git/tutorials/atlassian-git-cheatsheet}{Git cheatsheet}. \item You may also want to glance at Jess Johnson - \href{http://www.grokcode.com/717/how-to-use-source-control-effectively/}{How To Use Source Control Effectively}. -\item If your University subscribes to Lynda, they offer \href{https://www.lynda.com/search?q=git}{a number of Git courses} (with which I have no experience). +\item LinkedIn Learning offers \href{https://www.linkedin.com/learning/search?keywords=git}{a number of Git courses} (with which I have no experience). \end{itemize} @@ -183,4 +183,4 @@ \subsubsection{More notes on \texttt{git diff}} logbook/entries/researchinfrastructure.tex | 2 +- logbook/entries/tasks.tex | 2 +- 5 files changed, 104 insertions(+), 3 deletions(-) -\end{lstlisting} \ No newline at end of file +\end{lstlisting} diff --git a/logbook/entries/git_together.tex b/logbook/entries/git_together.tex index 868aa91..09d858b 100644 --- a/logbook/entries/git_together.tex +++ b/logbook/entries/git_together.tex @@ -112,38 +112,39 @@ \subsubsection{Branching and merging} git branch # List all of the branches in your repository. +git branch -a +# List all of the branches in your repository, including remote branches. + git branch # Create a new branch called . This does not check out the new branch. +git push -u origin +# Push the local branch to the remote repository (named origin). +# The -u option sets up tracking between the local and remote branches, so pull and push work without specifying the branch name. + git branch -d -# Delete the specified branch. This is a ``safe'' operation in that Git prevents -# you from deleting the branch if it has unmerged changes. +# Delete the specified branch. This is a `safe' operation in that Git prevents you from deleting the branch if it has unmerged changes. git branch -D -# Force delete the specified branch, even if it has unmerged changes. This is -# the command to use if you want to permanently throw away all of the commits -# associated with a particular line of development. +# Force delete the specified branch, even if it has unmerged changes. +# This is the command to use if you want to permanently throw away all of the commits associated with a particular line of development. git branch -m # Rename the current branch to . git checkout -# Check out the specified branch, which should have already been created with -# git branch. This makes the current branch, and updates the -# working directory to match. +# Check out the specified branch, which should have already been created with git branch. +# This makes the current branch, and updates the working directory to match. git checkout -b -# Create and check out . The -b option is a convenience flag that -# tells Git to run git branch before running git checkout -# . git checkout -b +# Create and check out . +# The -b option is a convenience flag that tells Git to run git branch before running git checkout . git merge -# Merge the specified branch into the current branch. Git will determine the -# merge algorithm automatically (discussed below). +# Merge the specified branch into the current branch. Git will determine the merge algorithm automatically (discussed below). git merge --no-ff -# Merge the specified branch into the current branch, but always generate a -# merge commit (even if it was a fast-forward merge). +# Merge the specified branch into the current branch, but always generate a merge commit (even if it was a fast-forward merge). \end{lstlisting} In some cases, you might forget to pull the latest commit before making a new one. diff --git a/logbook/entries/juliaintro.tex b/logbook/entries/juliaintro.tex index a5a9a5c..c13c008 100644 --- a/logbook/entries/juliaintro.tex +++ b/logbook/entries/juliaintro.tex @@ -1,13 +1,13 @@ Jonathan prefers Julia to Matlab. \subsubsection{Julia resources} -Jonathan learned Julia by starting with the \href{https://lectures.quantecon.org/jl/index_learning_julia.html}{QuantEcon introduction to Julia}. +Jonathan learned Julia by starting with the \href{https://julia.quantecon.org/intro.html}{QuantEcon introduction to Julia}. Here are some helpful resources: \begin{enumerate} \item \href{https://julialang.org}{julialang.org}: for installation and general info about julia: blogs, publications, conference -\item \href{https://docs.julialang.org/en/stable/manual/introduction/}{docs.julialang.org/en/stable/manual/introduction}: excellent manual for julia -\item \href{https://lectures.quantecon.org/jl/}{lectures.quantecon.org/jl}: an excellent manual for Julia with a macro vibe +\item \href{https://docs.julialang.org/en/v1/manual/getting-started/}{docs.julialang.org/en/v1/manual/getting-started/}: excellent manual for julia +\item \href{https://julia.quantecon.org/intro.html}{julia.quantecon.org/intro.html}: an excellent manual for Julia with a macro vibe \item \href{https://github.com/bkamins/Julia-DataFrames-Tutorial}{github.com/bkamins/Julia-DataFrames-Tutorial}: excellent tutorial on how to read/use DataFrames in Julia \item \href{https://www.juliabloggers.com/}{juliabloggers.com}: to keep up with advancements in Julia \item \href{http://www.johnmyleswhite.com/}{johnmyleswhite.com}: for interesting discussions about performance in Julia diff --git a/logbook/entries/logbookexplained.tex b/logbook/entries/logbookexplained.tex index 44420f0..fe73564 100644 --- a/logbook/entries/logbookexplained.tex +++ b/logbook/entries/logbookexplained.tex @@ -27,5 +27,5 @@ \subsection{Committing logbook entries} Doing so often creates merge conflicts when users preview their own logbook entries by compiling the logbook PDF locally. Similarly, do not commit the paper PDF to the repository. \item Do not commit the \texttt{logbook.aux}, \texttt{logbook.log}, nor \texttt{logbook.out} files generated by your \LaTeX compiler. - Either add these to your \texttt{.gitignore} file or have your Makefile delete these after succesful compilation of the PDF. + Either add these to your \texttt{.gitignore} file or have your Makefile delete these after successful compilation of the PDF. \end{itemize} diff --git a/logbook/entries/make.tex b/logbook/entries/make.tex index e03592a..5a510e0 100755 --- a/logbook/entries/make.tex +++ b/logbook/entries/make.tex @@ -25,7 +25,7 @@ \subsection{Learn Make} \item Mike Bostock: \href{https://bost.ocks.org/mike/make/}{Why Use Make} \item Karl Broman: \href{http://kbroman.org/minimal_make/}{minimal make} \item Kieran Healy: \href{http://plain-text.co/pull-it-together.html}{Pull It Together} (The Plain Person's Guide to Plain Text Social Science) -\item Zachary M. Jones: \href{http://zmjones.com/make/}{GNU Make for Reproducible Data Analysis} +\item Zachary M. Jones: \href{https://web.archive.org/web/20240222042605/http://zmjones.com/make/}{GNU Make for Reproducible Data Analysis} \end{itemize} \subsection{A live example} @@ -394,3 +394,11 @@ \subsection{Running Make} \item I am not sure about the best practice for the Torque scheduler. \end{itemize} +\subsection{Validating Makefiles} + +A semi-common mistake is to submit a Makefile that only worked on your machine because of the state of the files. +For example, an input existed on your machine, so you forgot to supply a rule to produce that input, +which others will need. +There is a shell script in the task \texttt{actions\_scripts} that executes \texttt{make -n} on every Makefile in the \texttt{tasks} folder to validate them. +This script is run automatically as a GitHub action when a pull request is submitted for review. +The script accepts a list of tasks as its argument if you want to only validate a subset of tasks. diff --git a/logbook/entries/manuscriptpreparation.tex b/logbook/entries/manuscriptpreparation.tex index 74e2935..884ebd4 100644 --- a/logbook/entries/manuscriptpreparation.tex +++ b/logbook/entries/manuscriptpreparation.tex @@ -10,7 +10,7 @@ \begin{itemize} \item -If you haven't installed \href{https://ctan.org/pkg/chktex?lang=en}{ChkTeX}, Macintosh installation instrutions are available at \url{http://www2.hawaii.edu/~ramonf/ChkTeXonMacOSX/index.html}. +If you haven't installed \href{https://ctan.org/pkg/chktex?lang=en}{ChkTeX}, Macintosh installation instructions are available at \url{http://www2.hawaii.edu/~ramonf/ChkTeXonMacOSX/index.html}. \item TeXtidote is available from \url{https://github.com/sylvainhalle/textidote}. To install on Mac OS X, install the \href{https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html}{JDK}. diff --git a/logbook/entries/taskassignments.tex b/logbook/entries/taskassignments.tex index c8563ed..3bb1d2d 100644 --- a/logbook/entries/taskassignments.tex +++ b/logbook/entries/taskassignments.tex @@ -8,59 +8,14 @@ You might try to use Slack as a project management tool, but I have not enjoyed it. It is better for free-flowing conversation than keeping a to-do list. -Relative to Asana, GitHub lacks due dates, subtasks, and arbitrary attachments. GitHub doesn't support issue-level due dates (GitLab does), -but we use GitHub's milestones to describe priorities and timetables. -GitHub doesn't allow hierarchical issues (i.e., subtasks), -but try checklists within GitHub issues. +but we use GitHub's milestones and labels to describe priorities and timetables. +You can add checklists and sub-issues within GitHub issues. GitHub limits the acceptable types of file attachments. - -Asana is organized into tasks, projects, and teams. -Tasks are the basic unit of work in Asana, -projects are lists of tasks, and -teams are groups of people that work together on tasks and projects. - -\begin{center} -\includegraphics[width=\textwidth]{./figures/workflow/asana_screenshot1.png} -\end{center} - -In this screenshot, -you see a list of projects in the left column of the ENSO workspace. -We're on the ``Presentation'' project right now, looking at a list of tasks. -One task is ``Slide improvements'', which is not assigned to anyone and lacks a due date. -It contains two subtasks: -one assigned to Kyle Meng without a due date -and -one assigned to Jonathan due April 9. -Both subtasks have been completed, so they are checked off and faded out. - -To better organize projects, you can divide tasks into ``sections''. -To create a section, just create a new task within a project and end the task name with a colon. - - -\begin{center} -\includegraphics[width=.8\textwidth]{./figures/workflow/asana_screenshot2.png} -\end{center} - -There is a comment thread associated with each task. -Asana conveniently records discussions that are categorized by task and searchable. -Kyle likely never saw the interaction in this screenshot because he was not a ``follower'' of this task. -That's good, because he was not interested in this task/conversation. -To make sure someone receives ``notifications'' related to a task in their inbox, add them as a follower. -The task assignee is automatically a follower (hence Kevin is following this task). -Commenters are automatically added as followers (hence Jonathan is also a follower). -You can grab someone's attention by using ``@'' and their name in a comment. - - -Jonathan loves the ``Inbox'' tab. -Following those notifications lets him track others' progress on tasks and alerts him to comments. -The ``My Tasks'' tab alerts him each day to tasks that are coming due soon. +Jonathan loves the GitHub ``Notifications'' tab. +Following those notifications lets him track others' progress and alerts him to comments. Jonathan recommends -(1) turning off Asana email notifications (Jonathan doesn't like email for workflow, though some of his collaborators do) +(1) turning off GitHub email notifications (Jonathan doesn't like email for workflow, though some of his collaborators do) and -(2) installing the \href{https://asana.com/apps/asana}{Asana app} on your smartphone (great way to see research progress while trapped in seminars!). - - - -To learn more about using Asana, see these \href{https://asana.com/guide/resources/get-started/quick-start#lessons?lesson=tasks-1}{introductory lessons}. \ No newline at end of file +(2) installing the \href{https://github.com/mobile}{GitHub app} on your smartphone (great way to see research progress while trapped in seminars!). diff --git a/logbook/entries/unixshelltips.tex b/logbook/entries/unixshelltips.tex index 907b08d..c3aef30 100644 --- a/logbook/entries/unixshelltips.tex +++ b/logbook/entries/unixshelltips.tex @@ -115,7 +115,7 @@ \subsection{Text editing on a Linux compute cluster} Most often, you'll be an environment where you get to choose your own text editor. However, in some (i.e., confidential) computing environments, you will not be free to install arbitrary software. -Nano, Emacs, and Vim will typically be installed everywhere, so it it worth knowing some basic info. +Nano, Emacs, and Vim will typically be installed everywhere, so it's worth knowing some basic info. Nano is the easiest tool. If you need to make a few edits to a modest file quickly without looking up the text editor's documentation, just use \href{https://www.nano-editor.org/docs.php}{nano}. @@ -155,7 +155,7 @@ \subsection{Multiple ``windows''} \subsection{SLURM (Simple Linux Utility for Resource Management)} SLURM is a job scheduler used on many computing clusters. -Read the \href{https://columbiauniversity.atlassian.net/wiki/spaces/rcs/pages/62145134/Insomnia+-+Submitting+Jobs}{Columbia HPC introduction} and then head over to the \href{https://slurm.schedmd.com/overview.html}{official documentation}. +Read the \href{https://columbiauniversity.atlassian.net/wiki/spaces/rcs/pages/62145124/Insomnia+HPC+Cluster+User+Documentation}{Columbia HPC introduction} and then head over to the \href{https://slurm.schedmd.com/overview.html}{official documentation}. This \href{https://slurm.schedmd.com/pdfs/summary.pdf}{two-page PDF} lists almost all the relevant commands you might need. \begin{itemize} @@ -233,7 +233,7 @@ \subsection{Jumping between MacOS GUI and Terminal} A few tips if you're using Terminal on your Mac but exclusively using the command line: \begin{itemize} \item You can drag the path of a Mac folder into Terminal (or Stata) by dragging the folder icon at the top of its Finder window into the Terminal prompt (\href{https://twitter.com/lukestein/status/1179792864630296578}{via Luke Stein}) - \item Typing \texttt{open .} in any directory in the Terminal will open that folder in the Finder (\href{https://twitter.com/FlorianOswald/status/1179779138892292096}{via Florian Oswald}) + \item Typing \texttt{open .} in any directory in the Terminal will open that folder in the Finder (via Florian Oswald's tweet) \end{itemize} \subsection{Other resources} diff --git a/logbook/figures/workflow/asana_screenshot1.png b/logbook/figures/workflow/asana_screenshot1.png deleted file mode 100644 index cfa0b30..0000000 Binary files a/logbook/figures/workflow/asana_screenshot1.png and /dev/null differ diff --git a/logbook/figures/workflow/asana_screenshot2.png b/logbook/figures/workflow/asana_screenshot2.png deleted file mode 100644 index feb87fd..0000000 Binary files a/logbook/figures/workflow/asana_screenshot2.png and /dev/null differ diff --git a/logbook/figures/workflow/make_on_win_gitbash_screenshot1.png b/logbook/figures/workflow/make_on_win_gitbash_screenshot1.png deleted file mode 100644 index 2f6cd33..0000000 Binary files a/logbook/figures/workflow/make_on_win_gitbash_screenshot1.png and /dev/null differ diff --git a/logbook/figures/workflow/make_on_win_gitbash_screenshot2.png b/logbook/figures/workflow/make_on_win_gitbash_screenshot2.png deleted file mode 100644 index 4239f36..0000000 Binary files a/logbook/figures/workflow/make_on_win_gitbash_screenshot2.png and /dev/null differ diff --git a/logbook/logbook.tex b/logbook/logbook.tex index f39352b..fcc1c93 100644 --- a/logbook/logbook.tex +++ b/logbook/logbook.tex @@ -29,6 +29,7 @@ \usepackage{etoc} %Package with \localtableofcontents \usepackage{multicol} \usepackage{bm} +\usepackage[T1]{fontenc} \definecolor{dkgreen}{rgb}{0,0.6,0} \definecolor{gray}{rgb}{0.5,0.5,0.5} diff --git a/paper/Makefile b/paper/Makefile index b7f0564..431e5ec 100644 --- a/paper/Makefile +++ b/paper/Makefile @@ -9,12 +9,18 @@ all: paper.pdf paper.pdf: paper.tex $(PAPER_INPUTS) if command -v sbatch > /dev/null ; then module load texlive; fi - pdflatex -draftmode paper.tex - bibtex paper.aux - pdflatex -draftmode paper.tex - pdflatex paper.tex - rm paper.log paper.aux paper.out - rm paper.bbl paper.blg + pdflatex -draftmode $< + bibtex $(basename $<).aux + pdflatex -draftmode $< + pdflatex $< + rm $(addprefix $(basename $<), .aux .log .out) + -rm $(addprefix $(basename $<), .bbl .blg) + +.PHONY: clean inputs +clean: + -rm *.aux *.log *.out + -rm *.toc *.bbl *.blg +inputs: $(PAPER_INPUTS) input/: mkdir $@ diff --git a/paper/paper.tex b/paper/paper.tex index b9384bd..74db7b2 100644 --- a/paper/paper.tex +++ b/paper/paper.tex @@ -38,7 +38,7 @@ \begin{document} -\bibliographystyle{../bib/aeanobold} +\bibliographystyle{../bib/aeanobold-oxford} % ----------------------------------------------- % ABSTRACT @@ -60,7 +60,7 @@ % BODY % ----------------------------------------------- -%\input{./sections/intro.tex} +\input{./sections/intro.tex} %\input{./sections/theory.tex} %\input{./sections/estimation.tex} %\input{./sections/discussion.tex} diff --git a/paper/sections/intro.tex b/paper/sections/intro.tex new file mode 100644 index 0000000..570244d --- /dev/null +++ b/paper/sections/intro.tex @@ -0,0 +1 @@ +Lorem ipsum \citep{DingelNeiman:2020}. diff --git a/slides/Makefile b/slides/Makefile index 8915af8..a51a4b8 100644 --- a/slides/Makefile +++ b/slides/Makefile @@ -1,28 +1,29 @@ #DEFINITIONS -slides_sections := $(shell grep '\\input' slides.tex | grep -v '%.*input' | grep -o 'sections/[A-Za-z0-9_\-]*\.tex') -slides_inputs := $(shell grep --no-filename 'input/' $(slides_sections) slides.tex | grep -v '^%' | grep -o 'input/[A-Za-z0-9_\.\-]*\.[a-z]*') +SLIDES_SECTIONS := $(shell grep '\\input' slides.tex | grep -v '%.*input' | grep -o 'sections/[A-Za-z0-9_\-]*\.tex') +SLIDES_INPUTS := $(shell grep --no-filename 'input/' $(SLIDES_SECTIONS) slides.tex | grep -v '^%' | grep -o 'input/[A-Za-z0-9_\.\-]*\.[a-z]*') all: slides.pdf +.PHONY: clean inputs clean: rm *.log *.aux +inputs: $(SLIDES_INPUTS) input/: mkdir $@ -slides.pdf: slides.tex $(slides_sections) $(slides_inputs) - #module load texlive +slides.pdf: slides.tex $(SLIDES_SECTIONS) $(SLIDES_INPUTS) + if command -v sbatch > /dev/null ; then module load texlive; fi pdflatex -draftmode $< - bibtex slides.aux + bibtex $(basename $<).aux pdflatex -draftmode $< pdflatex $< - rm slides.log slides.aux slides.out slides.toc - rm slides.snm slides.nav - rm slides.blg slides.bbl - #module unload texlive + rm $(addprefix $(basename $<),.log .aux .out .toc) + rm $(addprefix $(basename $<),.snm .nav) + rm $(addprefix $(basename $<),.blg .bbl) -slides_notesonly.pdf: slides.tex $(slides_sections) $(slides_inputs) +slides_notesonly.pdf: slides.tex $(SLIDES_SECTIONS) $(SLIDES_INPUTS) sed 's/notes=hide/notes=only/' slides.tex > slides_notesonly.tex pdflatex slides_notesonly.tex #This will fail due to natbib's bug with notes-only beamer rm slides_notesonly.tex slides_notesonly.log diff --git a/slides/sections/intro.tex b/slides/sections/intro.tex new file mode 100644 index 0000000..7428fda --- /dev/null +++ b/slides/sections/intro.tex @@ -0,0 +1,3 @@ +\begin{frame}{Introduction} +Lorem ipsum \citep{DingelNeiman:2020}. +\end{frame} diff --git a/slides/slides.tex b/slides/slides.tex index 83b42fd..22410a7 100644 --- a/slides/slides.tex +++ b/slides/slides.tex @@ -51,7 +51,7 @@ } \end{frame} %--------------------------------------------------------------------- -%\input{sections/intro.tex} +\input{sections/intro.tex} %--------------------------------------------------------------------- %--------------------------------------------------------------------- @@ -65,8 +65,8 @@ \appendix %\input{sections/appendix.tex} -%\bibliographystyle{../bib/aeanobold} -%\nobibliography{../bib/bib.bib} +\bibliographystyle{../bib/aeanobold-oxford} +\nobibliography{../bib/bib.bib} %\backupend diff --git a/tasks/check_makefiles/code/check_makefiles.sh b/tasks/actions_scripts/code/check_makefiles.sh similarity index 61% rename from tasks/check_makefiles/code/check_makefiles.sh rename to tasks/actions_scripts/code/check_makefiles.sh index c3509a0..e862714 100644 --- a/tasks/check_makefiles/code/check_makefiles.sh +++ b/tasks/actions_scripts/code/check_makefiles.sh @@ -5,12 +5,21 @@ # note: if no input is provided, # then all tasks with a Makefile will be checked. +# check if task graph is acyclic +make -C ../../task_graph/code ../output/graph.txt > /dev/null +CYCLIC=$(sed '1d; $d; s/-> //g; s/^[[:space:]]*//' ../../task_graph/output/graph.txt | tsort 2>&1 >/dev/null | sed 's/tsort: //' | sed 's/cycle in data/Cycle:/') +if [ -n "$CYCLIC" ]; then + echo "$CYCLIC" + exit 1 +fi + +# check makefiles TASKS="$@" # read in given tasks # if no task was given, run over all tasks with Makefiles if [ "$TASKS" == "" ] then - TASKS=$(find ../.. -name 'Makefile' | awk -F '/' '{print $3}') + TASKS=$(find ../.. -name 'Makefile' | sed "s|/code/Makefile||" | sed "s|\.\./\.\./||" | grep -v '^reports$') fi # default exit status diff --git a/tasks/check_makefiles/code/check_nontask_makefiles.sh b/tasks/actions_scripts/code/check_nontask_makefiles.sh similarity index 100% rename from tasks/check_makefiles/code/check_nontask_makefiles.sh rename to tasks/actions_scripts/code/check_nontask_makefiles.sh diff --git a/tasks/actions_scripts/code/check_tex_files.sh b/tasks/actions_scripts/code/check_tex_files.sh new file mode 100644 index 0000000..d296b57 --- /dev/null +++ b/tasks/actions_scripts/code/check_tex_files.sh @@ -0,0 +1,135 @@ +#!/bin/sh + +set -eo pipefail + +random_token() { + tr -dc A-Za-z0-9 /dev/null | head -c 32 + echo "" +} + +run() { + token="$(random_token)" + echo "::stop-commands::${token}" + echo -e "\033[1;34m${*@Q}\033[0m" + echo "::${token}::" + "$@" +} + +error() { + echo "::error :: $1" + exit 1 +} + +if [[ -n "$INPUT_TEXLIVE_VERSION" && -n "$INPUT_DOCKER_IMAGE" ]]; then + error "Input 'texlive_version' and 'docker_image' cannot co-exist". +fi + +export INPUT_TLMGR_REPO="" + +if [[ -z "$INPUT_DOCKER_IMAGE" ]]; then + INPUT_TLMGR_REPO="$INPUT_TEXLIVE_VERSION" + case "$INPUT_TEXLIVE_VERSION" in + "" | "latest" | "2024") + image_version="latest" + INPUT_TLMGR_REPO="latest" + ;; + "2023") + image_version="20240301" + ;; + "2022") + image_version="20230301" + ;; + "2021") + image_version="20220201" + ;; + "2020") + image_version="20210301" + ;; + *) + error "TeX Live version $INPUT_TEXLIVE_VERSION is not supported. The currently supported versions are 2020-2024 or latest." + ;; + esac + INPUT_DOCKER_IMAGE="ghcr.io/xu-cheng/texlive-full:$image_version" +fi + +# ref: https://docs.miktex.org/manual/envvars.html +run docker run --rm \ + -e "BIBINPUTS" \ + -e "BSTINPUTS" \ + -e "MFINPUTS" \ + -e "TEXINPUTS" \ + -e "TFMFONTS" \ + -e "HOME" \ + -e "INPUT_ROOT_FILE" \ + -e "INPUT_WORKING_DIRECTORY" \ + -e "INPUT_WORK_IN_ROOT_FILE_DIR" \ + -e "INPUT_CONTINUE_ON_ERROR" \ + -e "INPUT_COMPILER" \ + -e "INPUT_ARGS" \ + -e "INPUT_EXTRA_SYSTEM_PACKAGES" \ + -e "INPUT_EXTRA_FONTS" \ + -e "INPUT_PRE_COMPILE" \ + -e "INPUT_POST_COMPILE" \ + -e "INPUT_LATEXMK_SHELL_ESCAPE" \ + -e "INPUT_LATEXMK_USE_LUALATEX" \ + -e "INPUT_LATEXMK_USE_XELATEX" \ + -e "INPUT_TLMGR_REPO" \ + -e "GITHUB_JOB" \ + -e "GITHUB_REF" \ + -e "GITHUB_SHA" \ + -e "GITHUB_REPOSITORY" \ + -e "GITHUB_REPOSITORY_OWNER" \ + -e "GITHUB_REPOSITORY_OWNER_ID" \ + -e "GITHUB_RUN_ID" \ + -e "GITHUB_RUN_NUMBER" \ + -e "GITHUB_RETENTION_DAYS" \ + -e "GITHUB_RUN_ATTEMPT" \ + -e "GITHUB_REPOSITORY_ID" \ + -e "GITHUB_ACTOR_ID" \ + -e "GITHUB_ACTOR" \ + -e "GITHUB_TRIGGERING_ACTOR" \ + -e "GITHUB_WORKFLOW" \ + -e "GITHUB_HEAD_REF" \ + -e "GITHUB_BASE_REF" \ + -e "GITHUB_EVENT_NAME" \ + -e "GITHUB_SERVER_URL" \ + -e "GITHUB_API_URL" \ + -e "GITHUB_GRAPHQL_URL" \ + -e "GITHUB_REF_NAME" \ + -e "GITHUB_REF_PROTECTED" \ + -e "GITHUB_REF_TYPE" \ + -e "GITHUB_WORKFLOW_REF" \ + -e "GITHUB_WORKFLOW_SHA" \ + -e "GITHUB_WORKSPACE" \ + -e "GITHUB_ACTION" \ + -e "GITHUB_EVENT_PATH" \ + -e "GITHUB_ACTION_REPOSITORY" \ + -e "GITHUB_ACTION_REF" \ + -e "GITHUB_PATH" \ + -e "GITHUB_ENV" \ + -e "GITHUB_STEP_SUMMARY" \ + -e "GITHUB_STATE" \ + -e "GITHUB_OUTPUT" \ + -e "RUNNER_OS" \ + -e "RUNNER_ARCH" \ + -e "RUNNER_NAME" \ + -e "RUNNER_ENVIRONMENT" \ + -e "RUNNER_TOOL_CACHE" \ + -e "RUNNER_TEMP" \ + -e "RUNNER_WORKSPACE" \ + -e "ACTIONS_RUNTIME_URL" \ + -e "ACTIONS_RUNTIME_TOKEN" \ + -e "ACTIONS_CACHE_URL" \ + -e GITHUB_ACTIONS=true \ + -e CI=true \ + -v "/var/run/docker.sock":"/var/run/docker.sock" \ + -v "$HOME:$HOME" \ + -v "$GITHUB_ENV:$GITHUB_ENV" \ + -v "$GITHUB_OUTPUT:$GITHUB_OUTPUT" \ + -v "$GITHUB_STEP_SUMMARY:$GITHUB_STEP_SUMMARY" \ + -v "$GITHUB_PATH:$GITHUB_PATH" \ + -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" \ + -v "$GITHUB_ACTION_PATH/entrypoint.sh":/entrypoint.sh \ + -w "$GITHUB_WORKSPACE" \ + --entrypoint "./tasks/actions_scripts/code/entrypoint.sh" \ + "$INPUT_DOCKER_IMAGE" \ No newline at end of file diff --git a/tasks/actions_scripts/code/entrypoint.sh b/tasks/actions_scripts/code/entrypoint.sh new file mode 100755 index 0000000..d0b2bbf --- /dev/null +++ b/tasks/actions_scripts/code/entrypoint.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash + +set -eo pipefail +shopt -s extglob globstar nullglob + +info() { + echo -e "\033[1;34m$1\033[0m" +} + +warn() { + echo "::warning :: $1" +} + +error() { + echo "::error :: $1" + exit 1 +} + +# install git on old images +if ! command -v git &>/dev/null; then + apk --no-cache add git +fi +git config --system --add safe.directory "$GITHUB_WORKSPACE" + +if [[ -z "$INPUT_ROOT_FILE" ]]; then + error "Input 'root_file' is missing." +fi + +readarray -t root_file <<<"$INPUT_ROOT_FILE" + +if [[ -n "$INPUT_WORKING_DIRECTORY" ]]; then + if [[ ! -d "$INPUT_WORKING_DIRECTORY" ]]; then + mkdir -p "$INPUT_WORKING_DIRECTORY" + fi + cd "$INPUT_WORKING_DIRECTORY" +fi + +expanded_root_file=() +for pattern in "${root_file[@]}"; do + # shellcheck disable=SC2206 + expanded=($pattern) + expanded_root_file+=("${expanded[@]}") +done +root_file=("${expanded_root_file[@]}") + +if [[ -z "$INPUT_COMPILER" && -z "$INPUT_ARGS" ]]; then + warn "Input 'compiler' and 'args' are both empty. Reset them to default values." + INPUT_COMPILER="latexmk" + INPUT_ARGS="-pdf -file-line-error -halt-on-error -interaction=nonstopmode" +fi + +IFS=' ' read -r -a args <<<"$INPUT_ARGS" + +if [[ "$INPUT_COMPILER" = "latexmk" ]]; then + if [[ "$INPUT_LATEXMK_SHELL_ESCAPE" = "true" ]]; then + args+=("-shell-escape") + fi + + if [[ "$INPUT_LATEXMK_USE_LUALATEX" = "true" && "$INPUT_LATEXMK_USE_XELATEX" = "true" ]]; then + error "Input 'latexmk_use_lualatex' and 'latexmk_use_xelatex' cannot be used at the same time." + fi + + if [[ "$INPUT_LATEXMK_USE_LUALATEX" = "true" ]]; then + for i in "${!args[@]}"; do + if [[ "${args[i]}" = "-pdf" ]]; then + unset 'args[i]' + fi + done + args+=("-lualatex") + # LuaLaTeX use --flag instead of -flag for arguments. + for VAR in -file-line-error -halt-on-error -shell-escape; do + for i in "${!args[@]}"; do + if [[ "${args[i]}" = "$VAR" ]]; then + args[i]="-$VAR" + fi + done + done + args=("${args[@]/#-interaction=/--interaction=}") + fi + + if [[ "$INPUT_LATEXMK_USE_XELATEX" = "true" ]]; then + for i in "${!args[@]}"; do + if [[ "${args[i]}" = "-pdf" ]]; then + unset 'args[i]' + fi + done + args+=("-xelatex") + fi +else + for VAR in "${!INPUT_LATEXMK_@}"; do + if [[ "${!VAR}" = "true" ]]; then + error "Input '${VAR}' is only valid if input 'compiler' is set to 'latexmk'." + fi + done +fi + +if [[ -n "$INPUT_EXTRA_SYSTEM_PACKAGES" ]]; then + IFS=$' \t\n' + for pkg in $INPUT_EXTRA_SYSTEM_PACKAGES; do + info "Install $pkg by apk" + apk --no-cache add "$pkg" + done +fi + +if [[ -n "$INPUT_EXTRA_FONTS" ]]; then + readarray -t extra_fonts <<<"$INPUT_EXTRA_FONTS" + expanded_extra_fonts=() + for pattern in "${extra_fonts[@]}"; do + # shellcheck disable=SC2206 + expanded=($pattern) + expanded_extra_fonts+=("${expanded[@]}") + done + extra_fonts=("${expanded_extra_fonts[@]}") + + mkdir -p "$HOME/.local/share/fonts/" + + for f in "${extra_fonts[@]}"; do + if [[ -z "$f" ]]; then + continue + fi + + info "Install font $f" + cp -r "$f" "$HOME/.local/share/fonts/" + done + + fc-cache -fv +fi + +if [[ -n "$INPUT_TLMGR_REPO" && "$INPUT_TLMGR_REPO" != latest ]]; then + tlmgr_repo_url="https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/$INPUT_TLMGR_REPO/tlnet-final" + info "Set tlmgr repo to $tlmgr_repo_url" + tlmgr option repository "$tlmgr_repo_url" +fi + +if [[ -n "$INPUT_PRE_COMPILE" ]]; then + info "Run pre compile commands" + eval "$INPUT_PRE_COMPILE" +fi + +exit_code=0 + +for f in "${root_file[@]}"; do + if [[ -z "$f" ]]; then + continue + fi + + if [[ "$INPUT_WORK_IN_ROOT_FILE_DIR" = "true" ]]; then + pushd "$(dirname "$f")" >/dev/null + f="$(basename "$f")" + info "Compile $f in $PWD" + else + info "Compile $f" + fi + + if [[ ! -f "$f" ]]; then + error "File '$f' cannot be found from the directory '$PWD'." + fi + + "$INPUT_COMPILER" "${args[@]}" "$f" || ret="$?" + if [[ "$ret" -ne 0 ]]; then + if [[ "$INPUT_CONTINUE_ON_ERROR" = "true" ]]; then + exit_code="$ret" + else + exit "$ret" + fi + fi + + if [[ "$INPUT_WORK_IN_ROOT_FILE_DIR" = "true" ]]; then + popd >/dev/null + fi +done + +if [[ -n "$INPUT_POST_COMPILE" ]]; then + info "Run post compile commands" + eval "$INPUT_POST_COMPILE" +fi + +exit "$exit_code" diff --git a/tasks/check_makefiles/ReadMe.md b/tasks/actions_scripts/readme.md similarity index 59% rename from tasks/check_makefiles/ReadMe.md rename to tasks/actions_scripts/readme.md index 06bcc27..22d7ea3 100644 --- a/tasks/check_makefiles/ReadMe.md +++ b/tasks/actions_scripts/readme.md @@ -1,9 +1,7 @@ -# Check Makefiles +# Actions Scripts -This task includes bash scripts to verify the makefiles in the repo. -These are used by GitHub Actions. +This folder contains shell scripts that are used by our GitHub Actions. -## Code * `check_makefiles.sh`: This script runs `make -n` for each task listed. If an error occurs, the script exits with a non-zero exit code. @@ -11,3 +9,5 @@ If no task-list argument is provided, the script check all task folders. * `check_nontask_makefiles.sh`: This script runs `make -n` in the paper, slides, and logbook folders. If an error occurs, the script exits with a non-zero exit code. +* `check_tex_files.sh` and `entrypoint.sh` compiles the slides and paper. +These scripts were first developed by [Xu Chang](https://github.com/xu-cheng/latex-action). diff --git a/tasks/generic.make b/tasks/generic.make new file mode 100644 index 0000000..74b9f7e --- /dev/null +++ b/tasks/generic.make @@ -0,0 +1,19 @@ +OOPR = ../output ../temp ../report ../input $(if $(shell command -v sbatch),run.sbatch) #Order-only pre-reqs +JULIA_OOPR = $(OOPR) ../input/Manifest.toml ../input/Project.toml +STATA_OOPR = $(OOPR) profile.do + +slurmlogs ../input ../output ../temp ../report: + mkdir $@ + +run.sbatch: ../../setup_environment/code/run.sbatch | slurmlogs + ln -sf $< $@ +../input/Project.toml: ../../setup_environment/output/Project.toml | ../input/Manifest.toml ../input + ln -sf $< $@ +../input/Manifest.toml: ../../setup_environment/output/Manifest.toml | ../input + ln -sf $< $@ +profile.do: ../../setup_environment/code/profile.do + ln -sf $< $@ + +.PRECIOUS: ../../% +../../%: #Generic recipe to produce outputs from upstream tasks + $(MAKE) -C $(subst output/,code/,$(dir $@)) ../output/$(notdir $@) diff --git a/tasks/onboarding/code/Makefile b/tasks/onboarding/code/Makefile new file mode 100644 index 0000000..f8b9b59 --- /dev/null +++ b/tasks/onboarding/code/Makefile @@ -0,0 +1,43 @@ +R_VERSION = 4.4.0 # R version + +all: + echo This task has an empty default target. Please specify if you want to 'make commandlinetools' or something else. + echo Use 'make commandlinetools' to install Homebrew and tools like graphviz, wget, etc. + echo Use 'make basictex' to install BasicTeX and 'make ../output/TeX_packages.txt' to install TeX packages. + echo Use 'make julia' to install Julia via juliaup. + echo Use 'make R' to install R. + +.PHONY: commandlinetools basictex julia R + +commandlinetools: homebrew_installed.sh + bash homebrew_installed.sh + for pkg in graphviz wget ghostscript imagemagick; do \ + brew search "$$pkg" ;\ + brew install --ask "$$pkg" ;\ + done + +basictex: homebrew_installed.sh + bash homebrew_installed.sh + brew install --ask --cask basictex + +../output/TeX_packages.txt: TeX_packages.sh | ../output + command -v tlmgr >/dev/null 2>&1 || { echo "Error: 'tlmgr' is not installed or not in PATH."; exit 1; } + echo "The tlmgr command requires sudo privileges to run. It will ask for your password." + bash $< + +julia: + @if command -v juliaup >/dev/null 2>&1; then \ + echo "juliaup is already installed! Use 'juliaup' to manage Julia versions. Type 'juliaup --help' for more info."; \ + else \ + echo "juliaup not found. Installing Julia via juliaup..."; \ + curl -fsSL https://install.julialang.org | sh; \ + echo "Installation complete! Restart your shell or run:"; \ + echo " source ~/.bashrc # or ~/.zshrc depending on your shell"; \ + fi +R: ../output/R_installed.txt + @echo "R is already installed! Use 'R' to start the R console." +../output/R_installed.txt: install_R.sh | $(OOPR) + @bash $< $(R_VERSION) + +../output: + mkdir ../output diff --git a/tasks/onboarding/code/TeX_packages.sh b/tasks/onboarding/code/TeX_packages.sh new file mode 100644 index 0000000..977e627 --- /dev/null +++ b/tasks/onboarding/code/TeX_packages.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +#slides packages +sudo tlmgr install beamertheme-metropolis appendixnumberbeamer fmtcount datetime epstopdf +#paper packages +sudo tlmgr install appendix placeins physics etoc epstopdf footmisc chktex +#logbook packages +sudo tlmgr install bbm bbm-macros csvsimple + +echo 'Shell script to install TeX packages ran. Here is the list of installed TeX packages:' > ../output/TeX_packages.txt +tlmgr info --list --only-installed >> ../output/TeX_packages.txt diff --git a/tasks/onboarding/code/homebrew_installed.sh b/tasks/onboarding/code/homebrew_installed.sh new file mode 100644 index 0000000..09e3254 --- /dev/null +++ b/tasks/onboarding/code/homebrew_installed.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# This script checks if Homebrew is installed and, if not, installs it. + +# Check if Homebrew is installed +if command -v brew >/dev/null 2>&1; then + echo "Homebrew is already installed." +else + echo "Homebrew not found. Installing Homebrew..." + + # Use the official Homebrew installation command for macOS + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + + # Add Homebrew to PATH if it was just installed (for non-default shells or new installs) + if [[ -d /opt/homebrew/bin ]]; then + eval "$(/opt/homebrew/bin/brew shellenv)" + elif [[ -d /usr/local/bin ]]; then + eval "$(/usr/local/bin/brew shellenv)" + fi + + echo "Homebrew installation complete." +fi diff --git a/tasks/onboarding/code/install_R.sh b/tasks/onboarding/code/install_R.sh new file mode 100644 index 0000000..70996c0 --- /dev/null +++ b/tasks/onboarding/code/install_R.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +set -e # Exit on any error +set -u # Treat unset variables as errors + +# Read argument: R version +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi +R_VERSION="$1" + +OS="$(uname)" +echo "Operating system detected: $OS" + +if [[ "$OS" == "Darwin" ]]; then + ARCH=$(uname -m) + if [[ "$ARCH" == "arm64" ]]; then + CRAN_R_URL="https://cran.r-project.org/bin/macosx/big-sur-arm64/base/R-$R_VERSION-arm64.pkg" + elif [[ "$ARCH" == "x86_64" ]]; then + CRAN_R_URL="https://cran.r-project.org/bin/macosx/big-sur-x86_64/base/R-$R_VERSION-x86_64.pkg" + else + echo "Unsupported macOS architecture: $ARCH" + exit 1 + fi + + # Check if R is already installed + if command -v R >/dev/null 2>&1; then + echo "R is already installed at $(which R). Version:" + R --version + + read -p "Preferred version is $R_VERSION. Overwrite existing R installation? [confirmed/n] " answer + answer="${answer:-n}" # default to 'n' if empty + if [[ "$answer" != "confirmed" ]]; then + echo "Skipping installation." + exit 0 + fi + fi + + echo "Downloading R from $CRAN_R_URL" + curl -O "$CRAN_R_URL" + sudo installer -pkg "$(basename "$CRAN_R_URL")" -target / + echo "R installed successfully on macOS." + +elif [[ "$OS" == "Linux" ]]; then + # Check if R is already installed + if command -v R >/dev/null 2>&1; then + echo "R is already installed at $(which R). Version:" + R --version + + read -p "Preferred version is $R_VERSION. Overwrite existing R installation? [confirmed/n] " answer + answer="${answer:-n}" # default to 'n' if empty + if [[ "$answer" != "confirmed" ]]; then + echo "Skipping installation." + exit 0 + fi + fi + + echo "Installing R on Debian/Ubuntu Linux..." + + # Install prerequisites + sudo apt-get update + sudo apt-get install -y --no-install-recommends software-properties-common dirmngr gnupg apt-transport-https ca-certificates + + # Add CRAN GPG key and repo + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | gpg --dearmor | sudo tee /etc/apt/keyrings/cran.gpg > /dev/null + + # Get Ubuntu codename + CODENAME=$(lsb_release -cs) + echo "Ubuntu codename: $CODENAME" + + # Add CRAN to sources.list + echo "deb [signed-by=/etc/apt/keyrings/cran.gpg] https://cloud.r-project.org/bin/linux/ubuntu $CODENAME-cran40/" | sudo tee /etc/apt/sources.list.d/cran.list + + sudo apt-get update + sudo apt-get install -y --no-install-recommends r-base + + echo "R installed successfully on Linux." + +else + echo "Unsupported operating system: $OS" + exit 1 +fi + +# Show R version +echo "Installed R version:" +R --version + +# Write stamp file +STAMP_FILE="../output/R_installed.txt" + +echo "R version $R_VERSION installed on $(date)" > "$STAMP_FILE" +echo "Stamp file created at $STAMP_FILE" diff --git a/tasks/onboarding/readme.md b/tasks/onboarding/readme.md new file mode 100644 index 0000000..845663c --- /dev/null +++ b/tasks/onboarding/readme.md @@ -0,0 +1,20 @@ +Welcome. You've just joined our research team or just bought a new computer, so you need to set up software from scratch. +This task contains scripts to help you do that. + +We assume that you have a Mac. +`make` comes pre-installed on your Mac, so you can run `make` in this task folder (specifically, in `onboarding/code`). +Type `git --version` in the Terminal to confirm you have `git`, which you need to clone this repo, on your machine. +If you do not have `git`, install it as part the MacOS developer tools by running `xcode-select --install`. + +If you run `make commandlinetools` in `onboarding/code`, that script will install [Homebrew](https://brew.sh/), a package manager for Mac, +and use it to install common shell programs ([wget](https://www.gnu.org/software/wget/), [ImageMagick](https://imagemagick.org/), etc). + +If you run `make basictex`, it should install light-weight MacTeX and associated packages (but I haven't tested this yet myself). +See [BasicTeX](https://www.tug.org/mactex/morepackages.html). +I prefer a 117MB install to a [6GB install](https://www.tug.org/mactex/mactex-download.html), but you will need to use `tlmgr`, the TeX Live package manager, to install needed packages. Type `make ../output/TeX_packages.txt`. + +If you run `make julia`, it'll check to see whether you have `juliaup`. +If you do not, it installs `juliaup` from the web. + +You should also install the following software on your Mac: +- [VSCode](https://code.visualstudio.com/download) diff --git a/tasks/permissions/code/permissions.bash b/tasks/permissions/code/permissions.bash deleted file mode 100644 index 46926b6..0000000 --- a/tasks/permissions/code/permissions.bash +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -projectdirectory=/project/jdingel -projectgroup=pi-jdingel - -#Change directory ownership -echo "The following directories do not have group ${projectgroup}:" -find ${projectdirectory} -type d -not -group ${projectgroup} -echo "Changing group ownership to ${projectgroup}" -find ${projectdirectory} -type d -not -group ${projectgroup} -exec chgrp ${projectgroup} {} \; - -#Change file ownership -echo "The following files do not have group ${projectgroup}:" -find ${projectdirectory} -type f -not -group ${projectgroup} -echo "Changing group ownership to ${projectgroup}" -find ${projectdirectory} -type f -not -group ${projectgroup} -exec chgrp ${projectgroup} {} \; - -#Change directories' write permissions -echo "Group members do not have write permissions on the following directories" -find ${projectdirectory} -type d -group ${projectgroup} -not -perm -g=w -echo "Give group members write permissions" -find ${projectdirectory} -type d -group ${projectgroup} -not -perm -g=w -exec chmod 770 {} \; - -#Check sticky bits -echo "The sticky bit isn't set on the following directories:" -find ${projectdirectory} -type d -group ${projectgroup} -not -perm -2000 -echo "Set the stick bits on those directories:" -find ${projectdirectory} -type d -group ${projectgroup} -not -perm -2000 -exec chmod g+s {} \; diff --git a/tasks/setup_environment/code/profile.do b/tasks/setup_environment/code/profile.do new file mode 100644 index 0000000..a2ac957 --- /dev/null +++ b/tasks/setup_environment/code/profile.do @@ -0,0 +1,21 @@ +version 18 + +// If we are not in setup_environment: +if strpos(c(pwd),"setup_environment")==0 { + // First verify that adofile directory exists + capture confirm file ../../setup_environment/code/Stata_adofiles/ + // If it does not exist, throw error message + if _rc != 0 { + display as error /// + "ERROR: Please set up ../../setup_environment/code/Stata_adofiles/. " /// + "Because this error is reported within profile.do, the rest of the Stata script will nonetheless run. " /// + "The shell_function catcher should flag an error after the script finishes." + confirm file ../../setup_environment/code/Stata_adofiles/ + } + // Otherwise, set Stata_adofiles to beginning of Stata's adofile search path + adopath ++ "../../setup_environment/code/Stata_adofiles" + // Verify that stata package requirements file exists + confirm file ../../setup_environment/output/stata_requirements.txt + // Enforce package version requirements + require using "../../setup_environment/output/stata_requirements.txt" +} diff --git a/tasks/setup_environment/code/run.sbatch b/tasks/setup_environment/code/run.sbatch new file mode 100644 index 0000000..354cbad --- /dev/null +++ b/tasks/setup_environment/code/run.sbatch @@ -0,0 +1,12 @@ +#!/bin/sh + +#SBATCH --output=slurmlogs/%x_%j.out +#SBATCH --error=slurmlogs/%x_%j.err +#SBATCH --cpus-per-task=1 +#SBATCH --mem-per-cpu=5g +#SBATCH --time=3:00:00 +#SBATCH --mail-type=FAIL +#SBATCH --account=sscc + +$command1 +$command2 diff --git a/tasks/shell_functions.make b/tasks/shell_functions.make new file mode 100644 index 0000000..d26cb1e --- /dev/null +++ b/tasks/shell_functions.make @@ -0,0 +1,11 @@ +FUNCTIONS = $(shell cat ../../shell_functions.sh) +STATA = @$(FUNCTIONS); stata_with_flag +JULIA = @$(FUNCTIONS); julia_pc_and_slurm +R = @$(FUNCTIONS); R_pc_and_slurm + +#If 'make -n' option is invoked +ifneq (,$(findstring n,$(MAKEFLAGS))) +STATA := STATA +JULIA := JULIA +R := R +endif diff --git a/tasks/shell_functions.sh b/tasks/shell_functions.sh new file mode 100644 index 0000000..5f1d0ce --- /dev/null +++ b/tasks/shell_functions.sh @@ -0,0 +1,102 @@ +`##This file defines shell functions that +# 1. Improve Makefiles' readability by compartmentalizing all the "if" statement around SLURM vs local executables. +# 2. Cause Stata to report an error to Make when the Stata log file ends with an error.`; + +stata_with_flag() { + stata_pc_and_slurm $@; + LOGFILE_NAME=$(basename ${1%.*}.log); + if grep -q '^r([0-9]*);$' ${LOGFILE_NAME}; then + echo "STATA ERROR: There are errors in the running of ${1} file"; + echo "Exiting Status: $(grep '^r([0-9]*);$' ${LOGFILE_NAME} | head -1)"; + exit 1; + fi +} ; + +stata_pc_and_slurm() { + if command -v sbatch > /dev/null ; then + command1="module load stata/18"; + command2="stata-se -e $@"; + jobname1=$(echo "${1%.*}_" | sed 's/\.\.\/input\///'); + jobname2=$(echo ${@:2} | sed -E 's/\.\.\/(temp|input|code|output).//g' | sed -E 's/( |\/)/_/g' | cut -c '1-200'); + full_jobname=$(echo "$jobname1$jobname2" | sed -E 's/_{2,}/_/g' | sed -E 's/_$//g'); + print_info Stata $@; + sbatch -W --export=command1="$command1",command2="$command2" --job-name="$full_jobname" run.sbatch; + else + if command -v stata-mp >/dev/null ; then + print_info Stata $@; + stata-mp -e $@; + elif command -v stata-se >/dev/null ; then + print_info Stata $@; + stata-se -e $@; + else + echo "Stata/MP and Stata/SE not installed on this machine, or not found in PATH. Please fix to continue."; + exit 1; + fi; + fi +} ; + +R_with_flag() { + R_pc_and_slurm $@; + if [ "$1" == "--no-job-name" ]; then + shift; + fi ; + LOGFILE_NAME=$(basename ${1%.*}.log); + if grep -q '^r([0-9]*);$' ${LOGFILE_NAME}; then + echo "R ERROR: There are errors in the running of ${1} file"; + echo "Exiting Status: $(grep '^r([0-9]*);$' ${LOGFILE_NAME} | head -1)"; + exit 1; + fi +} ; + +R_pc_and_slurm() { + if command -v sbatch > /dev/null ; then + command1="echo R is not a module on Columbia HPC"; + command2="Rscript $@"; + jobname1=$(echo "${1%.*}_" | sed 's/\.\.\/input\///'); + jobname2=$(echo ${@:2} | sed -E 's/\.\.\/(temp|input|code|output).//g' | sed -E 's/( |\/)/_/g' | cut -c '1-200'); + full_jobname=$(echo "$jobname1$jobname2" | sed -E 's/_{2,}/_/g' | sed -E 's/_$//g'); + print_info R $@; + sbatch -W --export=command1="$command1",command2="$command2" --job-name="$full_jobname" run.sbatch; + else + print_info R $@; + Rscript $@; + fi +} ; + +julia_pc_and_slurm() { + if command -v sbatch > /dev/null ; then + command1="module load julia/1.10.2"; + command2="julia $@"; + jobname1=$(echo "${1%.*}_" | sed 's/\.\.\/input\///'); + jobname2=$(echo ${@:2} | sed -E 's/\.\.\/(temp|input|code|output).//g' | sed -E 's/( |\/)/_/g' | cut -c '1-200'); + full_jobname=$(echo "$jobname1$jobname2" | sed -E 's/_{2,}/_/g' | sed -E 's/_$//g'); + print_info Julia $@; + sbatch -W --export=command1="$command1",command2="$command2" --job-name="$full_jobname" run.sbatch; + else + print_info Julia $@; + check_julia_version 1.10.2; + julia +1.10.2 $@; + fi +} ; + +check_julia_version() { + if command -v juliaup > /dev/null; then + if ! juliaup status | grep -v "release" | grep -q "$1"; then + echo "Julia version $1 was not found on this machine. To use version $1, please run 'juliaup add $1'."; + exit 1; + fi; + else + echo "juliaup is not installed on this machine. Please follow instructions to do so at https://julialang.org/downloads/"; + exit 1; + fi; +} ; + +print_info() { + software=$1; + shift; + if [ $# == 1 ]; then + echo "Running ${1} via ${software}, waiting..."; + else + echo "Running ${1} via ${software} with args = ${@:2}, waiting..."; + fi +} diff --git a/tasks/task_graph/code/Makefile b/tasks/task_graph/code/Makefile index 15c7267..b28f2e5 100755 --- a/tasks/task_graph/code/Makefile +++ b/tasks/task_graph/code/Makefile @@ -1,17 +1,23 @@ SHELL=bash all: ../output/task_flow.png +branch: $(addprefix ../output/,task_flow.png task_flow_branchdiff.png task_flow_branchdiff_trim.png) -../output: - mkdir $@ +include ../../generic.make ../output/graph.txt: graphmaker.sh $(shell find ../../ -name "Makefile") | ../output bash graphmaker.sh +../output/graph_branchdiff.txt: ../output/graph.txt + cat <(grep -v '}' ../output/graph.txt) <(git diff --name-status origin/main HEAD | grep '^[AM]' | grep -o 'tasks/[A-Za-z0-9_]*' | sed 's/tasks\///' | sort | uniq | sed 's/$$/ [shape=box]/') <(echo }) > $@ +../output/graph_%_trim.txt: graph_box_trim.sh ../output/graph_%.txt + bash $^ $@ + ../output/task_flow.png: ../output/graph.txt dot -Grankdir=LR -Tpng $< -o $@ -../output/graph_branchdiff.txt: ../output/graph.txt - cat <(grep -v '}' ../output/graph.txt) <(git diff --name-status master HEAD | grep '^[AM]' | grep -o 'tasks/[A-z0-9_]*' | sed 's/tasks\///' | sort | uniq | sed 's/$$/ [shape=box]/') <(echo }) > $@ -../output/task_flow_branchdiff.png: ../output/graph_branchdiff.txt +../output/task_flow_%.png: ../output/graph_%.txt dot -Grankdir=LR -Tpng $< -o $@ + +../output/code_dependencies.txt: code_dependencies.sh $(shell find ../../ -name "Makefile") | ../output + bash code_dependencies.sh ../output/graph_bidirectional_edges.txt: ../output/graph.txt grep -v '{\|}' ../output/graph.txt | awk -F'->' '{if ($$2 < $$1) {print $$2 "," $$1} else {print $$1 "," $$2} }' | sort | uniq -d > $@ diff --git a/tasks/task_graph/code/code_dependencies.sh b/tasks/task_graph/code/code_dependencies.sh new file mode 100644 index 0000000..d22b363 --- /dev/null +++ b/tasks/task_graph/code/code_dependencies.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#Script to document code dependencies + +find ../../*/code -maxdepth 1 -name "Makefile" | while read makefile; do + # Extract the current task name from the Makefile path + current_task=$(echo "$makefile" | sed 's/\.\.\/\.\.\///g' | sed 's/\/code\/Makefile//') # current task that Makefile is located + + # In current_task's Makefile, search for ../taskname/code patterns + grep -oh '\.\./[A-Za-z0-9_]*/code' "$makefile" | \ + # Ignore interactive_fe_estimation: setup_environment Makefile mentions this task for user_specific_sbatch + grep -v '\.\./interactive_fe_estimation' | \ + + # Cleanup path name; just get task + sed 's/\.\.\///g' | sed 's/\/code//g' | \ + + while read -r dependency_task; do + if [[ "$current_task" == "task_graph" && "$dependency_task" == "output" ]]; then + continue + fi + echo "$dependency_task -> $current_task" + done + +done | sort | uniq >> ../output/code_dependencies.txt diff --git a/tasks/task_graph/code/graph_box_trim.sh b/tasks/task_graph/code/graph_box_trim.sh new file mode 100644 index 0000000..3736dcf --- /dev/null +++ b/tasks/task_graph/code/graph_box_trim.sh @@ -0,0 +1,32 @@ +#!/bin/bash +#Script to generate graph of tasks based on specified terminal nodes denoted by [shape=box] +#This script simply starts at the terminal nodes and takes one step upstream repeatedly until it encounters no upstream pre-requisites. +#Input path is $1; output path is $2 + +grep -v '^#' $1 | sed 's/^[[:space:]]*//' | sed 's/ \-> /\->/' > temp_graph.txt #Remove comments from graph. + +ENDPOINTS="$(grep '\[shape=box\]' temp_graph.txt | sed 's/ \[shape=box\]/$|/' | sed 's/^/\->/' | tr -d '\n' | sed 's/|$//')" +echo $ENDPOINTS + +PREV_ENDPOINTS="" +while [ -n "$ENDPOINTS" ] +do +awk /${ENDPOINTS}/{print} temp_graph.txt >> temp.txt +PREV_ENDPOINTS="$ENDPOINTS" +ENDPOINTS=$(awk /${ENDPOINTS}/{print} temp_graph.txt | awk -F'->' '{print "->" $1 "$|"}' | sort | uniq | tr -d '\n' | sed 's/|$//') +if [ "$ENDPOINTS" = "$PREV_ENDPOINTS" ]; then + break +fi +done +cat <(echo -e 'digraph G {') <(sort temp.txt | uniq | grep -v 'setup_environment' | sed 's/\-\>/ \-\> /') <(grep '\[shape=box\]' temp_graph.txt) <(echo '}') > $2 +rm temp.txt temp_graph.txt + +# Check if trimmed graph is acyclic +grep -v '\[shape=box\]' $2 > temp_trim.txt +CYCLIC=$(sed '1d; $d; s/-> //g; s/^[[:space:]]*//' temp_trim.txt | tsort 2>&1 >/dev/null | sed 's/tsort: //' | sed 's/cycle in data/Cycle:/') +if [ -n "$CYCLIC" ]; then + echo "$CYCLIC" + rm temp_trim.txt + exit 1 +fi +rm temp_trim.txt