initial commit

.github/actions/setup-venv/action.yml

@@ -0,0 +1,63 @@
+name: Python virtualenv
+description: Set up a Python virtual environment with caching
+inputs:
+  python-version:
+    description: The Python version to use
+    required: true
+  cache-prefix:
+    description: Update this to invalidate the cache
+    required: true
+    default: v0
+  torch-version:
+    description: The PyTorch version to install
+    required: false
+    default: '==2.5.0'
+runs:
+  using: composite
+  steps:
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - shell: bash
+      run: |
+        # Install prerequisites.
+        pip install --upgrade pip setuptools build wheel virtualenv
+
+    - shell: bash
+      run: |
+        # Get the exact Python version to use in the cache key.
+        echo "PYTHON_VERSION=$(python --version)" >> $GITHUB_ENV
+
+    - uses: actions/cache@v2
+      id: virtualenv-cache
+      with:
+        path: .venv
+        key: ${{ inputs.cache-prefix }}-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ inputs.torch-version }}-${{ hashFiles('*requirements.txt', '*pyproject.toml') }}
+        restore-keys: |
+          ${{ inputs.cache-prefix }}-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ inputs.torch-version }}
+
+    - if: steps.virtualenv-cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        # Set up virtual environment without cache hit.
+        test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
+        . .venv/bin/activate
+        pip install 'torch${{ inputs.torch-version }}' --index-url https://download.pytorch.org/whl/cpu
+        pip install -e .[all]
+
+    - if: steps.virtualenv-cache.outputs.cache-hit == 'true'
+      shell: bash
+      run: |
+        # Set up virtual environment from cache hit.
+        . .venv/bin/activate
+        pip install --no-deps -e .[all]
+
+    - shell: bash
+      run: |
+        # Show environment info.
+        . .venv/bin/activate
+        echo "✓ Installed $(python --version) virtual environment to $(which python)"
+        echo "========= Python packages ==========="
+        pip freeze

.github/workflows/main.yml

@@ -0,0 +1,136 @@
+name: Main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*.*.*'
+
+env:
+  # Change this to invalidate existing cache.
+  CACHE_PREFIX: v0
+  PYTHONPATH: ./src/
+
+jobs:
+  checks:
+    name: ${{ matrix.task.name }}
+    runs-on: [ubuntu-latest]
+    timeout-minutes: 5
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.10']
+        task:
+          - name: Lint
+            run: make lint-check
+
+          - name: Test
+            run: |
+              pytest -v --color=yes --durations=3 src/test/
+
+          - name: Type check
+            run: make type-check
+
+          - name: Build
+            run: make build
+
+          - name: Style
+            run: make style-check
+
+        include:
+          - python: '3.9'
+            task:
+              name: Lint (min Python)
+              run: make lint-check
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-venv
+        with:
+          python-version: ${{ matrix.python }}
+          cache-prefix: ${{ env.CACHE_PREFIX }}
+
+      - name: Restore mypy cache
+        if: matrix.task.name == 'Type check'
+        uses: actions/cache@v3
+        with:
+          path: .mypy_cache
+          key: mypy-${{ env.CACHE_PREFIX }}-${{ runner.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-${{ github.ref }}-${{ github.sha }}
+          restore-keys: |
+            mypy-${{ env.CACHE_PREFIX }}-${{ runner.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-${{ github.ref }}
+            mypy-${{ env.CACHE_PREFIX }}-${{ runner.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
+
+      - name: ${{ matrix.task.name }}
+        run: |
+          . .venv/bin/activate
+          ${{ matrix.task.run }}
+
+      - name: Upload package distribution files
+        if: matrix.task.name == 'Build'
+        uses: actions/upload-artifact@v3
+        with:
+          name: package
+          path: dist
+
+      - name: Clean up
+        if: always()
+        run: |
+          . .venv/bin/activate
+          pip uninstall -y ai2-olmo-eval
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: [checks]
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-venv
+        with:
+          python-version: '3.10'
+          cache-prefix: ${{ env.CACHE_PREFIX }}
+
+      - name: Prepare environment
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
+          echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+
+      - name: Download package distribution files
+        uses: actions/download-artifact@v3
+        with:
+          name: package
+          path: dist
+
+      - name: Generate release notes
+        run: |
+          . .venv/bin/activate
+          python src/scripts/release_notes.py > ${{ github.workspace }}-RELEASE_NOTES.md
+
+      - name: Publish package to PyPI
+        run: |
+          . .venv/bin/activate
+          twine upload -u __token__ -p '${{ secrets.PYPI_TOKEN }}' dist/*
+
+      - name: Publish GitHub release
+        uses: softprops/action-gh-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          body_path: ${{ github.workspace }}-RELEASE_NOTES.md
+          prerelease: ${{ contains(env.TAG, 'rc') }}
+          files: |
+            dist/*

.gitignore

@@ -0,0 +1,59 @@
+# build artifacts
+
+.eggs/
+.mypy_cache
+*.egg-info/
+build/
+dist/
+pip-wheel-metadata/
+private/
+tmp/
+
+# dev tools
+
+.envrc
+.python-version
+.idea
+.venv/
+.vscode/
+/*.iml
+pyrightconfig.json
+.ruff.toml
+
+
+# jupyter notebooks
+
+.ipynb_checkpoints
+
+
+# miscellaneous
+
+.cache/
+doc/_build/
+*.swp
+.DS_Store
+
+
+# python
+
+*.pyc
+*.pyo
+__pycache__
+
+
+# testing and continuous integration
+
+.coverage
+.pytest_cache/
+.benchmarks
+
+# documentation build artifacts
+
+docs/build
+site/
+
+# runs
+/runs/
+/wandb/
+/scratch/
+core

CHANGELOG.md

@@ -0,0 +1,12 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## Unreleased
+
+### Added
+
+- Added in-loop evals from original OLMo repo.

Makefile

@@ -0,0 +1,24 @@
+.PHONY : checks
+checks : style-check lint-check type-check
+
+.PHONY : style-check
+style-check :
+	@echo "======== running isort... ========"
+	@isort --check .
+	@echo "======== running black... ========"
+	@black --check .
+
+.PHONY : lint-check
+lint-check :
+	@echo "======== running ruff... ========="
+	@ruff check .
+
+.PHONY : type-check
+type-check :
+	@echo "======== running mypy... ========="
+	@mypy src/
+
+.PHONY : build
+build :
+	rm -rf *.egg-info/
+	python -m build