diff --git a/.github/workflows/lectures.yml b/.github/workflows/lectures.yml index 9c4ea3a8..f0eeffb8 100644 --- a/.github/workflows/lectures.yml +++ b/.github/workflows/lectures.yml @@ -33,4 +33,5 @@ jobs: channel-priority: strict - name: Render lectures run: | - snakemake -d lectures/ -s lectures/Snakefile -c 1 \ No newline at end of file + cd lectures + snakemake -c 1 --use-conda \ No newline at end of file diff --git a/lectures/Snakefile b/lectures/Snakefile index 56fc042f..3897874b 100644 --- a/lectures/Snakefile +++ b/lectures/Snakefile @@ -38,6 +38,8 @@ rule render_jupyter: "{f}.ipynb" output: "{f}.html" + conda: + "envs/jupyter.yml" shell: """ jupyter nbconvert --to slides --execute {input} @@ -49,6 +51,8 @@ rule render_jupyter_pdf: "{f}.ipynb" output: "{f}.pdf" + conda: + "envs/jupyter.yml" shell: """ jupyter nbconvert --to webpdf {input} --allow-chromium-download diff --git a/lectures/environment.yml b/lectures/environment.yml index c5bef063..98dd4db6 100644 --- a/lectures/environment.yml +++ b/lectures/environment.yml @@ -3,22 +3,9 @@ channels: - conda-forge - bioconda dependencies: - # Snakemake - snakemake-minimal - - # Jupyter - - python - - jupyter - - pandas - - jupyter_contrib_nbextensions - - nbconvert - - seaborn - - rpy2 - - pyppeteer - - jinja2 - - # R and Xaringan + # R - phantomjs - r-base - r-ggplot2 @@ -28,6 +15,3 @@ dependencies: - r-tidyr - r-xaringan - r-webshot - - pip - - pip: - - RISE diff --git a/lectures/envs/jupyter.yml b/lectures/envs/jupyter.yml new file mode 100644 index 00000000..dfcc7c5e --- /dev/null +++ b/lectures/envs/jupyter.yml @@ -0,0 +1,16 @@ +name: jupyter-lectures +channels: + - conda-forge +dependencies: + - python + - jupyter + - jupyterlab + - pandas + - nbconvert + - seaborn + - rpy2 + - pyppeteer + - jinja2 + - pip + - pip: + - RISE \ No newline at end of file diff --git a/lectures/putting-it-together/putting-it-together.Rmd b/lectures/putting-it-together/putting-it-together.Rmd index 3fa33c9c..553b22c0 100644 --- a/lectures/putting-it-together/putting-it-together.Rmd +++ b/lectures/putting-it-together/putting-it-together.Rmd @@ -69,21 +69,70 @@ By working reproducibly you will also make your life a lot easier! -- -* Many software support the “project way of working”, *e.g.* .green[Rstudio] and the text editors .green[Sublime Text] and .green[Atom] +* Use .green[Git] to version control your projects -- -* Use .green[Git] to create structured and version controlled project repositories +* Do not store data and results/output in your Git repository ---- +-- -# Everything can be a project +* When in doubt, commit often instead of seldom -Project directory templates, *e.g.* NBIS project template: +--- + +# Find your own project structure + +.pull-left[ +.small[An example .green[Snakemake]-based project:] +```no-highlight +project/ + ├── code/ + ├── data/ + │ ├── meta/ + │ ├── raw_external/ + │ └── raw_internal/ + ├── doc/ + ├── intermediate/ + ├── logs/ + ├── notebooks/ + │ └── Untitled.ipynb + ├── results/ + │ ├── figures/ + │ ├── reports/ + │ └── tables/ + ├── scratch/ + ├── .gitignore + ├── config.yml + ├── environment.yml + ├── Dockerfile + ├── README.md + └── Snakefile +``` +] -.center[] +.pull-right[ +.small[An example .green[Nextflow]-based project:] +```no-highlight +project/ + ├── bin/ + │ └── report.qmd + ├── data/ + │ └── metadata.csv + ├── doc/ + ├── env/ + │ ├── Dockerfile + │ └── environment.yml + ├── results/ + ├── .gitignore + ├── main.nf + ├── nextflow.config + └── README.md +``` +] * https://github.com/NBISweden/project_template + * https://github.com/fasterius/nbis-support-template * https://github.com/snakemake-workflows/snakemake-workflow-template --- @@ -117,7 +166,7 @@ Project directory templates, *e.g.* NBIS project template: * Avoid generating files .green[interactively] or doing things .green[by hand] - - there is no way to track how they were made + - There is no way to track how they were made -- @@ -137,43 +186,18 @@ Project directory templates, *e.g.* NBIS project template: .pull-left[
-.green[Minimal]: write code in a reproducible way - -] - --- - -.pull-right[ - -Connect your results with the code: -* Use .green[R Markdown] documents or .green[Jupyter] notebooks - -Take another step: -* Convert your code into a .green[Snakemake] / .green[Nextflow] workflow - -] - ---- - -# What is reasonable for your project? - -.pull-left[
- -.green[Minimal]: write code in a reproducible way - -.green[Good]: versioned and structured repository - +.green[Minimal]: write code in a reproducible way and track your environment ] -- .pull-right[ +* Track your projects with a .green[Git] repository each; publish code along + with your results on *e.g.* -Use .green[Git] for version controlling and collaboration: -* Create one .green[Git] repository per project -* Track your changes with .green[Git] -* Publish your code along with your results on *e.g.* - +* Use .green[Conda] to install software in environments that can be exported and + installed on a different system; also publish your `environment.yml` file + along with your code ] --- @@ -182,21 +206,16 @@ Use .green[Git] for version controlling and collaboration: .pull-left[
-.green[Minimal]: write code in a reproducible way - -.green[Good]: versioned and structured repository - -.green[Better]: organize software dependencies +.green[Minimal]: write code in a reproducible way and track your environment +.green[Good]: structure and document your code with notebooks ] -- .pull-right[ - -Manage your depencencies: -* Use .green[Conda] to install software in environments that can be easily exported and installed on a different system - +* Use .green[R Markdown] or .green[Jupyter] notebooks to better keep track of + and document your code ] --- @@ -205,23 +224,20 @@ Manage your depencencies: .pull-left[
-.green[Minimal]: write code in a reproducible way +.green[Minimal]: write code in a reproducible way and track your environment -.green[Good]: versioned and structured repository - -.green[Better]: organize software dependencies - -.green[Best]: export everything! +.green[Good]: structure and document your code with notebooks +.green[Great]: track the _full_ environment and connect your code in a workflow ] -- .pull-right[ +* Go one step beyond in tracking your environment using .green[Docker] and/or + .green[Singularity/Apptainer] -Completely recreate the compute system: -* Consider packaging your project inside a or together with a .green[Docker] or .green[Singularity] container - +* Convert your code into a .green[Snakemake] / .green[Nextflow] workflow ] --- @@ -264,10 +280,16 @@ Completely recreate the compute system: .green[Literate programming] -* **Jupyter** – Create and share notebooks in a variety of languages and formats by using a web browser. -* **R Markdown** – Developed by Rstudio, focuses on generating high-quality documents. -* **Zeppelin** – Developed by Apache. Closely integrated with Spark for distributed computing and Big Data applications. -* **Beaker** – Newcomer based on Ipython, just as Jupyter. Has a focus on integrating multiple languages in the same notebook. +* **Jupyter** – Create and share notebooks in a variety of languages and formats + by using a web browser. +* **R Markdown** – Developed by Posit (previously Rstudio), focused on + generating high-quality documents. +* **Quarto** - Dveloped by Posit (previously RStudio), command-line tool focused + on generating high-quality documents in a language-agnostic way +* **Zeppelin** – Developed by Apache. Closely integrated with Spark for + distributed computing and Big Data applications. +* **Beaker** – Newcomer based on Ipython, just as Jupyter. Has a focus on + integrating multiple languages in the same notebook. --- @@ -295,6 +317,6 @@ class: center, middle Any questions related to reproducible research tools and concepts? Talk to an NBIS expert! -* Online (.green[zoom]) +* Online (.green[Zoom]) * Every .green[Tuesday, 14.00-15.00] (except public holidays) -* Check .green[www.nbis.se/events] for zoom link and more info +* Check .green[www.nbis.se/events] for Zoom link and more info diff --git a/lectures/putting-it-together/putting-it-together.pdf b/lectures/putting-it-together/putting-it-together.pdf index 0d80b1e6..54970ed5 100644 Binary files a/lectures/putting-it-together/putting-it-together.pdf and b/lectures/putting-it-together/putting-it-together.pdf differ