diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a157c00 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +# --- This Project ---# +data/ +logs/ +.env + +# Docker secrets +/secrets/ +*.secret +*.key +*.pem +*.crt + +# --- OS & Temp Files --- +.DS_Store +Thumbs.db +*.swp +*.swo +*.tmp +*~ +~$* + + diff --git a/compose.override.yml b/compose.override.yml new file mode 100644 index 0000000..bec2ca3 --- /dev/null +++ b/compose.override.yml @@ -0,0 +1,39 @@ +services: + jupyterhub: + container_name: ${COMPOSE_PROJECT_NAME}-jupyterhub + environment: + - POSTGRES_HOST=postgres + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + + - JUPYTERHUB_CRYPT_KEY=${JUPYTERHUB_CRYPT_KEY} + - DOCKER_NETWORK_NAME=${DOCKER_NETWORK_NAME} + - NBGRADER_DB_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${NBGRADER_DB} + ports: + - ${JUPYTERHUB_PORT:-8000}:8000 + volumes: + - ./data/jupyter:/srv/jupyterhub/data + networks: + - jupyterhub-network + depends_on: + - postgres + + postgres: + container_name: ${COMPOSE_PROJECT_NAME}-postgres + ports: + - ${POSTGRES_PORT:-5432}:5432 + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - jupyterhub-network + + nginx: + container_name: ${COMPOSE_PROJECT_NAME}-nginx + ports: + - ${NGINX_HTTP_PORT:-80}:80 + - ${NGINX_HTTPS_PORT:-443}:443 + networks: + - jupyterhub-network + depends_on: + - jupyterhub diff --git a/compose.yml b/compose.yml index e69de29..f46cef8 100644 --- a/compose.yml +++ b/compose.yml @@ -0,0 +1,10 @@ +include: + - jupyterhub/compose.yml + - postgres/compose.yml + - nginx/compose.yml + +networks: + jupyterhub-network: + name: ${DOCKER_NETWORK_NAME:-jupyterhub-network} + driver: bridge + attachable: true diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..5ca09a9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1750776420, + "narHash": "sha256-/CG+w0o0oJ5itVklOoLbdn2dGB0wbZVOoDm4np6w09A=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "30a61f056ac492e3b7cdcb69c1e6abdcf00e39cf", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..d5c8c56 --- /dev/null +++ b/flake.nix @@ -0,0 +1,161 @@ +{ + description = "Docker JupyterHub development environment"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils, ... }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + + app = { + setup-dev = pkgs.writeShellScriptBin "setup-dev" '' + echo "Setting up basic Project Structure" + echo "----------------------------------" + echo "Setting up logs directory" + mkdir -p logs + mkdir -p logs/nginx + mkdir -p logs/jupyterhub + mkdir -p logs/postgres + echo "Logs Directory:" + tree logs/ + echo "" + echo "Setting up jupyter-data" + mkdir -p jupyter-data/nbgrader/courses/example-course/{autograded,feedback,release,source/assigment1,submitted} + mkdir -p jupyter-data/nbgrader/{exchange,users} + cat > jupyter-data/nbgrader/courses/example-course/source/assigment1/sample.ipynb << 'EOL' + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sample Notebook\n", + "## Created with Bash\n", + "\n", + "This Jupyter notebook was generated automatically using a bash script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Python code example\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "x = np.linspace(0, 10, 100)\n", + "y = np.sin(x)\n", + "\n", + "plt.plot(x, y)\n", + "plt.title('Sine Wave')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps\n", + "\n", + "- Run the code cells\n", + "- Add more cells\n", + "- Experiment with different Python libraries" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 + } + EOL + echo "jupyter-data Directory:" + tree jupyter-data + ''; + + storage = pkgs.writeShellScriptBin "storage" '' + echo "Creating storage" + mkdir -p storage/{jupyter,postgres} + chmod -R 644 storage/ + ''; + + build = pkgs.writeShellScriptBin "build" '' + echo "Building all Containers" + ''; + + clean = pkgs.writeShellScriptBin "clean" '' + echo "Cleaning Project" + echo "----------------------------------" + echo "Removing logs" + rm -rf logs/ + echo "Removing jupyter-data" + rm -rf jupyter-data/ + ''; + + reset = pkgs.writeShellScriptBin "reset" '' + ${app.clean}/bin/clean + ${app.setup-dev}/bin/setup-dev + ''; + }; + in + { + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + # For deployment/testing + docker + docker-compose + + # Terminal Tools + tree + ] ++ (builtins.attrValues app); + + shellHook = '' + echo "JupyterHub dev shell" + echo "------------------------------------" + echo "$(docker --version)" + echo "------------------------------------" + echo "Available commands:" + echo " setup-dev - Set Up dev environment" + echo " clean - Clean the Project" + echo " reset - Reset project completly (Use with caution it whipes all generated Data)" + ''; + }; + + apps = { + setup-dev = { + type = "app"; + program = "${app.setup-dev}/bin/setup-dev"; + }; + + build = { + type = "app"; + program = "${app.build}/bin/build"; + }; + + default = self.apps.${system}.dev; + }; + }); +} diff --git a/hub/postgres/compose.yml b/hub/postgres/compose.yml deleted file mode 100644 index b7fc090..0000000 --- a/hub/postgres/compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -services: - db: - image: postgres - restart: always - environment: - POSTRGES_PASSWORD: 123456 - volumes: - - pgdata:/var/lib/postgresql/data - - adminer: - image: adminer - restart: always - ports: - - 8080:8080 - -volumes: - pgdata: diff --git a/hub/jhub/compose.yml b/jupyter-share/compose.yml similarity index 100% rename from hub/jhub/compose.yml rename to jupyter-share/compose.yml diff --git a/jupyterhub/Dockerfile b/jupyterhub/Dockerfile new file mode 100644 index 0000000..c2bc488 --- /dev/null +++ b/jupyterhub/Dockerfile @@ -0,0 +1,48 @@ +FROM quay.io/jupyterhub/jupyterhub:latest + +# Install system dependencies +USER root +RUN apt-get update && apt-get install -y \ + curl \ + gnupg \ + iproute2 \ + iputils-ping \ + netcat \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install Docker CLI (needed for DockerSpawner) +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu focal stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update \ + && apt-get install -y docker-ce-cli \ + && rm -rf /var/lib/apt/lists/* + +# Install Required packages +RUN pip install --no-cache-dir \ + dockerspawner \ + jupyterhub-nativeauthenticator \ + jupyterhub-dummyauthenticator \ + nbgrader \ + psycopg2-binary + +RUN mkdir -p /srv/nbgrader/exchange +RUN chmod -R 777 /srv/nbgrader/exchange +RUN mkdir -p /srv/nbgrader/courses +RUN chown -R 1000:100 /srv/nbgrader + +#RUN jupyter nbextension install --sys-prefix --py nbgrader --overwrite +#RUN jupyter nbextension enable --sys-prefix --py nbgrader +#RUN jupyter serverextension enable --sys-prefix --py nbgrader + +# Set Workdir +WORKDIR /srv/jupyterhub + +# Generate Cookie Secret +RUN openssl rand -hex 32 > /srv/jupyterhub/cookie_secret +RUN chmod 600 /srv/jupyterhub/cookie_secret +#ENV JUPYTERHUB_AUTH_TOKEN=$(openssl rand -hex 32) + +EXPOSE 8000 + +CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"] diff --git a/jupyterhub/compose.yml b/jupyterhub/compose.yml new file mode 100644 index 0000000..89cc6f3 --- /dev/null +++ b/jupyterhub/compose.yml @@ -0,0 +1,15 @@ +services: + jupyterhub: + build: . + container_name: jupyterhub + restart: unless-stopped + ports: + - 8000:8000 + environment: + - JUPYTERHUB_LOG_LEVEL=DEBUG + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./templates:/srv/jupyterhub/templates:ro + - ./jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py + - ./nbgrader_config.py:/srv/nbgrader/courses/nbgrader_config.py + - ../logs/jupyterhub:/var/log/jupyterhub diff --git a/jupyterhub/conf_test.py b/jupyterhub/conf_test.py new file mode 100644 index 0000000..345e513 --- /dev/null +++ b/jupyterhub/conf_test.py @@ -0,0 +1,9 @@ +c = get_config() + +c.JupyterHub.authenticator_class = "dummy" +c.JupyterHub.spawner_class = "docker" +c.JupyterHub.hub_ip = '0.0.0.0' +c.JupyterHub.hub_connect_ip = 'jupyterhub' +c.DockerSpawner.image = 'jupyter/base-notebook' +c.DockerSpawner.network_name = 'stack_default' +c.DockerSpawner.remove = True diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py new file mode 100644 index 0000000..16b09f2 --- /dev/null +++ b/jupyterhub/jupyterhub_config.py @@ -0,0 +1,227 @@ +import os +import sys +from dockerspawner import DockerSpawner +from jupyterhub.utils import random_port +from nativeauthenticator import NativeAuthenticator +#from nbgrader.auth import JupyterHubAuthPlugin +from traitlets import Unicode, Bool, Int, Float + +import secrets +import string +from datetime import datetime, timedelta +from typing import Optional + +def generate_api_token( + length: int = 32, + prefix: Optional[str] = None, +) -> dict: + """ + Generate a secure random API token with optional prefix and expiration. + + Args: + length: Length of the token (default: 32) + prefix: Optional prefix for the token (e.g., 'hub_') + expiration_days: Optional expiration in days from now + + Returns: + - token: The generated token + """ + # Generate cryptographically secure random string + alphabet = string.ascii_letters + string.digits + token = ''.join(secrets.choice(alphabet) for _ in range(length)) + + # Add prefix if specified + if prefix: + token = f"{prefix}{token}" + + return token + +import subprocess + +def is_service_available_cmd(host, port): + """Check service using system commands""" + try: + subprocess.run( + ['nc', '-z', host, str(port)], + check=True, + timeout=3, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + return True + except (subprocess.TimeoutExpired, subprocess.CalledProcessError): + return False + +# Configuration file for JupyterHub +c = get_config() + +# Base URL configuration (The new way, dont use ip/port/bind_url attributes it throws a warning and is deprecated) +port = int(os.environ.get('JUPYTERHUB_PORT', '8000')) +base_url = os.environ.get('JUPYTERHUB_BASE_URL', '/jupyter/') +c.JupyterHub.bind_url = f"http://0.0.0.0:{port}{base_url}" + +# Database configuration +c.JupyterHub.db_url = f"postgresql://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}@stack-{os.environ['POSTGRES_HOST']}:5432/{os.environ['POSTGRES_DB']}" +#c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite' + + +# Authenticator configuration - NativeAuthenticator +c.JupyterHub.authenticator_class = NativeAuthenticator + +# Native Authenticator settings +#c.NativeAuthenticator.create_system_users = True +c.NativeAuthenticator.minimum_password_length = int(os.environ.get('NATIVE_AUTH_MIN_PASSWORD_LENGTH', '8')) +c.NativeAuthenticator.check_common_password = True +c.NativeAuthenticator.enable_signup = True +c.NativeAuthenticator.ask_email_on_signup = False +c.NativeAuthenticator.allow_2fa = False + +# Admin configuration +admin_user = os.environ.get('JUPYTERHUB_ADMIN_USER', 'admin') +c.Authenticator.admin_users = {admin_user, 'DerGrumpf'} +c.Authenticator.any_allow_config = True +c.Authenticator.allow_all = True + +# Custom logo and templates +c.JupyterHub.logo_file = '/srv/jupyterhub/templates/logo.png' +c.JupyterHub.template_paths = ['/etc/jupyterhub/templates'] + + +# Hub environment for containers +c.JupyterHub.spawner_class = DockerSpawner +c.Spawner.ip = '0.0.0.0' +c.DockerSpawner.image = os.environ.get('NOTEBOOK_IMAGE', 'jupyter/scipy-notebook:latest') +'''network_name = os.environ.get('DOCKER_NETWORK_NAME', 'stack_default') +c.DockerSpawner.network_name = network_name +c.DockerSpawner.extra_host_config = { + 'network_mode': network_name +} +c.DockerSpawner.remove = True +c.DockerSpawner.debug = True +c.DockerSpawner.default_url = os.environ.get('SPAWNER_DEFAULT_URL', '/lab') +c.DockerSpawner.use_internal_ip = True + +# Resources +c.DockerSpawner.mem_limit = os.environ.get('NOTEBOOK_MEMORY_LIMIT', '500M') +c.DockerSpawner.cpu_limit = float(os.environ.get('NOTEBOOK_CPU_LIMIT', '1.0')) + +# Volume +c.DockerSpawner.volumes = { + '/var/run/docker.sock': '/var/run/docker.sock' +# './data/jupyter/users/{username}': '/home/jovyan/work', +# './data/jupyter/nbgrader/exchange': '/srv/nbgrader/exchange', +# './data/jupyter/nbgrader/courses': '/srv/nbgrader/courses', +} + +c.DockerSpawner.notebook_dir = '/home/jovyan/work' + +# Container Environment +c.DockerSpawner.environment = { + 'GRANT_SUDO': '0', + 'JUPYTER_ENABLE_LAB': os.environ.get('ENABLE_LAB', '1'), + 'JUPYTERHUB_SINGLEUSER_APP': 'jupyter_server.serverapp.ServerApp' +} + +#c.DockerSpawner.hub_ip_connect = os.environ.get('HUB_IP', 'jupyterhub') +#c.DockerSpawner.hub_port_connect = 8081 + +def pre_spawn_hook(spawner): + """Create user directories before spawning""" + username = spawner.user.name + user_dir = f"./data/jupyter/users/{username}" + + import os + import stat + + if not os.path.exists(user_dir): + os.makedirs(user_dir, mode=0o755, exist_ok=True) + os.chown(user_dir, 1000, 1000) + + # TODO Nbgrader dirs + +c.DockerSpawner.pre_spawn_hook = pre_spawn_hook +''' + +''' +# Services configuration for NBGrader +c.JupyterHub.services = [ + { + 'name': 'nbgrader-formgrader', + 'url': 'http://127.0.0.1:9999', + 'command': [ + 'jupyter-nbgrader', + '--port=9999', + '--no-browser', + '--log-level=INFO' + ], + 'cwd': '/srv/jupyterhub/courses', + 'user': 'root', + 'environment': { + 'PYTHONPATH': '/srv/nbgrader', + 'NBGRADER_CONFIG_FILE': '/srv/jupyterhub/nbgrader/nbgrader_config.py' + } + } +] +''' +''' +# NBGrader configuration +c.JupyterHub.load_groups = { + 'nbgrader-instructors': ['instructor'], + 'nbgrader-students': [] +} +''' +''' +# NBGrader Config +c.JupyterHub.services = [ + { + 'name': 'nbgrader-formgrader', + 'url': 'http://127.0.0.1:9999', + 'api_token': generate_api_token(prefix='nbgrader'), + 'command': [ + 'python', '-m', 'nbgrader', 'formgrader', + #'--port=9999', + #'--no-browser', + '--log-level=INFO', + #'--base_url=/jupyter/services/nbgrader-formgrader', + '--debug' + ], + 'environment': { + 'JUPYTERHUB_SERVICE_URL': 'http://127.0.0.1:8081/jupyter/hub/api', + 'NBGRADER_CONFIG_FILE': '/srv/nbgrader/nbgrader_config.py', + 'PYTHONPATH': '/srv/nbgrader' + }, + 'cwd': '/srv/nbgrader/courses', + 'user': 'root' + } +] +''' + +# Security settings +c.JupyterHub.cookie_secret_file = '/srv/jupyterhub/cookie_secret' +c.ConfigurableHTTPProxy.auth_token = os.environ.get('JUPYTERHUB_AUTH_TOKEN', '') + +# Logging +c.JupyterHub.log_level = os.environ.get('LOG_LEVEL', 'INFO') + +# Shutdown settings +c.JupyterHub.shutdown_on_logout = False +c.JupyterHub.cleanup_servers = True + +# Timeout settings +c.JupyterHub.active_server_limit = 0 +c.JupyterHub.concurrent_spawn_limit = 10 + +# Allow named servers +c.JupyterHub.allow_named_servers = True +c.JupyterHub.named_server_limit_per_user = 3 + +print("JupyterHub configuration loaded successfully!") +print(f"Base URL: {c.JupyterHub.base_url}") +print(f"Bind URL: {c.JupyterHub.bind_url}") +print(f"Database URL: {c.JupyterHub.db_url}") +print(f"Docker Network: {c.DockerSpawner.network_name}") +print(f"Docker Image: {c.DockerSpawner.image}") +print("Postgres available", is_service_available_cmd("postgres", 5432)) + +with open("/srv/jupyterhub/cookie_secret") as f: + print(f.readlines()) diff --git a/jupyterhub/nbgrader/nbgrader_config.py b/jupyterhub/nbgrader/nbgrader_config.py new file mode 100644 index 0000000..0e8e798 --- /dev/null +++ b/jupyterhub/nbgrader/nbgrader_config.py @@ -0,0 +1,30 @@ +# /srv/nbgrader/nbgrader_config.py +c = get_config() + +# 1. Course Configuration +c.CourseDirectory.course_id = "intro-to-python" # Change to your course name +c.CourseDirectory.root = '/srv/nbgrader/courses' + +# 2. Exchange Directory (for submitting/collecting assignments) +c.Exchange.root = '/srv/nbgrader/exchange' +c.Exchange.path_includes_course = True + +# 3. JupyterHub Integration +#c.NbGrader.hub_url = 'http://jupyterhub:8081/hub/api' +#c.NbGrader.hubapi_token = 'your-hub-token' # Generate with: openssl rand -hex 32 + +# 4. Database Configuration +c.StudentAssignmentNotebook.database_url = 'sqlite:////srv/nbgrader/nbgrader.sqlite' + +# 5. Assignment Policies +c.ExecutePreprocessor.timeout = 300 # 5 minutes timeout per cell +c.ClearSolutions.code_stub = "# YOUR CODE HERE" +c.ClearSolutions.text_stub = "YOUR ANSWER HERE" + +# 6. Formgrader UI Settings +c.FormgradeApp.port = 9999 +c.FormgradeApp.authenticator_class = 'nbgrader.auth.hubauth.HubAuth' +c.FormgradeApp.ip = '0.0.0.0' + +# 7. Permissions +c.CourseDirectory.groupshared = True diff --git a/jupyterhub/nbgrader_config.py b/jupyterhub/nbgrader_config.py new file mode 100644 index 0000000..0e8e798 --- /dev/null +++ b/jupyterhub/nbgrader_config.py @@ -0,0 +1,30 @@ +# /srv/nbgrader/nbgrader_config.py +c = get_config() + +# 1. Course Configuration +c.CourseDirectory.course_id = "intro-to-python" # Change to your course name +c.CourseDirectory.root = '/srv/nbgrader/courses' + +# 2. Exchange Directory (for submitting/collecting assignments) +c.Exchange.root = '/srv/nbgrader/exchange' +c.Exchange.path_includes_course = True + +# 3. JupyterHub Integration +#c.NbGrader.hub_url = 'http://jupyterhub:8081/hub/api' +#c.NbGrader.hubapi_token = 'your-hub-token' # Generate with: openssl rand -hex 32 + +# 4. Database Configuration +c.StudentAssignmentNotebook.database_url = 'sqlite:////srv/nbgrader/nbgrader.sqlite' + +# 5. Assignment Policies +c.ExecutePreprocessor.timeout = 300 # 5 minutes timeout per cell +c.ClearSolutions.code_stub = "# YOUR CODE HERE" +c.ClearSolutions.text_stub = "YOUR ANSWER HERE" + +# 6. Formgrader UI Settings +c.FormgradeApp.port = 9999 +c.FormgradeApp.authenticator_class = 'nbgrader.auth.hubauth.HubAuth' +c.FormgradeApp.ip = '0.0.0.0' + +# 7. Permissions +c.CourseDirectory.groupshared = True diff --git a/jupyterhub/templates/login.html b/jupyterhub/templates/login.html new file mode 100644 index 0000000..e044846 --- /dev/null +++ b/jupyterhub/templates/login.html @@ -0,0 +1,74 @@ +{% extends "templates/page.html" %} + +{% block login %} +
Don't have an account? Sign up
+Already have an account? Sign in
+