diff --git a/.emacs/29.3/straight/versions/default.el b/.emacs/29.3/straight/versions/default.el new file mode 100644 index 0000000..abd651e --- /dev/null +++ b/.emacs/29.3/straight/versions/default.el @@ -0,0 +1,13 @@ +(("dash.el" . "39d067b9fbb2db65fc7a6938bfb21489ad990cb4") + ("el-get" . "c0713e8d8e8ad987fe1283d76b9c637a10f048ef") + ("emacs-buttercup" . "a1a86b027ffe030e1c78a9f43c50cd20a6fed19a") + ("emacsmirror-mirror" . "8cbbdaa750c897d05ee71980834699a7d7c2d208") + ("f.el" . "de6d4d40ddc844eee643e92d47b9d6a63fbebb48") + ("gnu-elpa-mirror" . "3d0759ef4792b6461f2979a4e70e1c819df7283a") + ("melpa" . "3126dac37def07fcaa667a325ee79349fb80d285") + ("nongnu-elpa" . "a9a649210a8d8b9295b5a1d0c7b60a77db03c14c") + ("org" . "071c6e986c424d2e496be7d0815d6e9cd83ae4e6") + ("org-ml" . "7a7b1e918e8440f3f6ddb37db9bd1471d0dad37d") + ("s.el" . "4d7d83122850cf70dc60662a73124f0be41ad186") + ("straight.el" . "88e574ae75344e39b436f863ef0344135c7b6517")) +:gamma diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad5e547..d53bd74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,33 +7,25 @@ jobs: strategy: matrix: emacs_version: - - '27.1' - - '27.2' - - '28.1' + - '29.3' steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - architecture: 'x64' - - uses: purcell/setup-emacs@master - with: - version: ${{ matrix.emacs_version }} - - uses: conao3/setup-cask@master + + - uses: mamba-org/setup-micromamba@v1.9.0 with: - version: '0.8.4' + micromamba-version: '1.5.6-0' + environment-file: env-${{ matrix.emacs_version }}.yml + cache-environment: true + post-cleanup: 'all' - - name: Setup testing databases - run: docker-compose up -d + - uses: hoverkraft-tech/compose-action@v2.0.1 + with: + up-flags: "--build" + down-flags: "--volumes" - name: Run tests - if: matrix.allow_failure != true + shell: micromamba-shell {0} run: | - cask install + export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libc_malloc_debug.so + make install make - - - name: Run tests (allow failure) - if: matrix.allow_failure == true - run: | - cask install - make || true diff --git a/.gitignore b/.gitignore index d86a94e..12a76bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.elc -.cask \ No newline at end of file +.emacs/*/straight/build +.emacs/*/straight/repos +.emacs/*/straight/build-cache.el \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ad51a38..00a3070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 4.0.0 + +breaking changes +- drop support for lots of unsupported database versions (technically a breaking + change, hence major bump) + +new features +- timestamps in headlines are now pulled into database (thanks aviav) + +bugfixes +- fix compilation and documentation errors (thanks aviav) +- fix whitespace issues with go-sqlcmd + ## 3.0.4 - fix compiler errors and warnings diff --git a/Makefile b/Makefile index 08cd431..53b9f20 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,4 @@ -CASK ?= cask -EMACS ?= emacs +EMACS ?= emacs -Q --batch --load init.el -l org-sql.el all: test @@ -9,24 +8,35 @@ test: ${MAKE} compile docs: - ${CASK} exec ${EMACS} -Q -batch -L . \ + ${EMACS} \ -l doc/org-sql-doc.el \ -f create-docs-file \ -f org-sql-create-all-erds stateless: - ${CASK} exec buttercup -L . -l test/org-sql-test-stateless.el + ${EMACS} -l test/org-sql-test-stateless.el -f buttercup-run-discover stateful: - ${CASK} exec buttercup -L . -l test/org-sql-test-stateful.el + ${EMACS} -l test/org-sql-test-stateful.el -f buttercup-run-discover compile: - ${CASK} build + ${EMACS} build ${MAKE} stateless ${MAKE} stateful ${MAKE} clean-elc clean-elc: - ${CASK} clean-elc + ${EMACS} clean-elc + +# install all development packages for the current version +install: + ${EMACS} --eval '(print "Install finished")' + +# write lockfile for current emacs version given each repo dependency +freeze: + ${EMACS} -f straight-freeze-versions + +thaw: + ${EMACS} -f straight-thaw-versions .PHONY: all test unit diff --git a/README.md b/README.md index 11e41e3..e64d4ef 100644 --- a/README.md +++ b/README.md @@ -38,41 +38,53 @@ One can also use `use-package` to automate this entire process ## Dependencies -Only Emacs 27.1+ has been tested +Only Emacs 29.3 has been tested. It will probably work for others. ### Emacs packages -- org-ml.el -- dash.el -- s.el -- f.el +- org-ml.el (5.8.8) +- dash.el (2.19.1) +- s.el (1.13) +- f.el (0.20.0) + +Versions indicated those that have been tested. Others may work but are not +guaranteed. + +As of version 5.8.8, org-ml.el requires org 9.6.x to work. 9.7.x and later +*will* break. ### Database Clients Only the client binary for your desired implementation are required (ensure they are in your PATH): -- sqlite3 +- sqlite3 - psql (PostgreSQL) - mysql (MariaDB/MySQL) - sqlcmd (SQL-Server) +See the conda environment file at `env-XX.Y.yml` (where XX.Y corresponds to the +emacs version) for the exact versions of the each client used for testing. These +were provided with the following packages: + +- sqlite (for sqlite) +- postgresql (for PostgreSQL) +- mysqlclient (for MariaDB and MySQL) +- go-sqlcmd (for SQL-Server) + + ### Database Servers The following databases servers/versions are supported and tested: -- PostgreSQL (13, 12, 11, 10, 9) -- MariaDB (10.5, 10.4, 10.3, 10.2) -- MySQL (8.0) -- SQL-Server (2019, 2017) +- PostgreSQL (16, 15, 14, 13) +- MariaDB (11.4, 10.6, 10.5) +- MySQL (8.4, 8.0) +- SQL-Server (2022, 2019, 2017) Many versions besides these will likely work; these are simply those that are in the testing suite. -MySQL 5.7 should also work for all functions except `org-sql-pull-from-db`, -which will throw mysterious syntax errors because this function relies on -recursive queries (which don't exist in 5.7). - # Configuration ## General Behavior @@ -404,31 +416,48 @@ Contributions welcome! But please take advantage of the testing environment (especially when contributing to code which directly interacts with the database servers). -## Dependencies +## Reproducible Environment -In addition to all required dependencies above: +The entire development environment is designed to be self-contained and +reproducible. All binaries (including emacs itself) are specified in a conda +environment, the databases are specified in a docker-compose file, and the +dependencies for emacs are specified in a `straight.el` profile in +`.emacs/XX.Y/straight/versions/default.el`. -- [cask](https://github.com/cask/cask) -- docker -- docker-compose -- [erd](https://github.com/BurntSushi/erd) -- make +The only prerequisites for running this are a working conda and docker-compose +installation. +It isn't stricly necessary to use this, but doing so ensures that all tests run +in a standardized manner across all machines and therefore minimizes 'bit rot +bugs'. -## Emacs setup +### Conda dependencies -Install all emacs dependencies: +Assuming a working mamba installation, install all binary dependencies and +activate: ``` sh -cask install --dev +mamba env create -f env-XX.Y.yml +conda activate env-XX.Y.yml ``` -## Test environment setup +### Emacs dependencies + +Run the following + +``` sh +export LD_PRELOAD=/usr/lib/libc_malloc_debug.so +make install +``` + +Note, the LD_PRELOAD setting is necessary if one gets and error about undefined +symbols for malloc_set_state. This is necessary for all `make ...` commands. + +### Database setup Except for SQLite, the each database for testing is encoded and set up using -`docker-compose` (see the included `docker-compose.yml` and -`docker-compose.override.yml` files). These are -necessary to run the stateful tests above. +`docker-compose` (see the included `docker-compose.yml` file). These are +necessary to run the stateful tests (see below). To set up the environment, start the docker-daemon (may require sudo). @@ -436,13 +465,15 @@ To set up the environment, start the docker-daemon (may require sudo). docker-compose up -d -V ``` +Add `--build` to rebuild images if altered (see below). + To shut down the environment: ``` sh docker-compose down ``` -### Dockerfile/Docker-compose Layout +#### Dockerfile/Docker-compose Layout Customization of the `docker-compose` files should not be necessary except when adding a new database for testing (or a new version). The 'base' docker images @@ -453,7 +484,7 @@ is set to the latest version of the container to pull. Note that MariaDB and MySQL are assumed to share the exact same container configuration, and thus they share the same Dockerfile. -## Running tests +### Running tests Tests are divided into stateless (pure functions, don't rely on external database implementations) and stateful (impure, interacts with the database @@ -483,7 +514,7 @@ Run all tests using both interpreted and compiled code: make test ``` -## Building documentation +### Building documentation To generate documentation: @@ -491,7 +522,39 @@ To generate documentation: make docs ``` +requires [erd](https://github.com/BurntSushi/erd) (which is unfortunately not in +conda). + +## Interactive development + +To use Emacs to edit the code of `org-sql`, one has several options, from most +to least reliable. + +### Emacs from conda + +Simply activate the conda environment set up from above and run emacs from the +shell. This will have all the required dependencies, but also won't have your +personal setup. + +### Personal emacs with straight dependencies + +More clunkily, if one wants/needs the exact versions of each emacs package, one +can copy the hashes from `emacs.d/XX.Y/straight/versions/default.el` into their +own straight config (or make a new profile). + +### Personal emacs config with cask + +Install [cask](https://github.com/cask/cask) and run the following (without the +conda env activated): + +``` sh +cask install +``` + +Run emacs as normal, and activate the conda environment to run the tests. + # Acknowledgements -The idea for this is based on [John Kitchin's](http://kitchingroup.cheme.cmu.edu/blog/2017/01/03/Find-stuff-in-org-mode-anywhere/) +The idea for this is based on [John +Kitchin's](http://kitchingroup.cheme.cmu.edu/blog/2017/01/03/Find-stuff-in-org-mode-anywhere/) implementation, which uses `emacsql` as the SQL backend. diff --git a/docker-compose.yml b/docker-compose.yml index 3fdaf8c..1bb0420 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,47 +11,37 @@ services: - POSTGRES_PASSWORD=toor - POSTGRES_USER=root ports: - - 60013:5432 - - psql-12: - <<: *psql - build: - context: ./test/docker/postgres - args: - IMAGE: postgres:12.6-alpine - container_name: org-sql-postgres-12 - ports: - - 60012:5432 + - 60016:5432 - psql-11: + psql-15: <<: *psql build: context: ./test/docker/postgres args: - IMAGE: postgres:11.11-alpine - container_name: org-sql-postgres-11 + IMAGE: postgres:15.8-alpine + container_name: org-sql-postgres-15 ports: - - 60011:5432 + - 60015:5432 - psql-10: + psql-14: <<: *psql build: context: ./test/docker/postgres args: - IMAGE: postgres:10.16-alpine - container_name: org-sql-postgres-10 + IMAGE: postgres:14.13-alpine + container_name: org-sql-postgres-14 ports: - - 60010:5432 + - 60014:5432 - psql-9: + psql-13: <<: *psql build: context: ./test/docker/postgres args: - IMAGE: postgres:9.6.21-alpine - container_name: org-sql-postgres-9 + IMAGE: postgres:13.16-alpine + container_name: org-sql-postgres-13 ports: - - 60009:5432 + - 60013:5432 ## mariadb @@ -63,38 +53,38 @@ services: environment: - MYSQL_ROOT_PASSWORD=toor ports: - - 60105:3306 + - 60114:3306 - mariadb-104: + mariadb-1011: <<: *mariadb build: context: ./test/docker/mariadb args: - IMAGE: mariadb:10.4.17 - container_name: org-sql-mariadb-104 + IMAGE: mariadb:10.11.8 + container_name: org-sql-mariadb-1011 ports: - - 60104:3306 + - 60111:3306 - mariadb-103: + mariadb-106: <<: *mariadb build: context: ./test/docker/mariadb args: - IMAGE: mariadb:10.3.17 - container_name: org-sql-mariadb-103 + IMAGE: mariadb:10.6.18 + container_name: org-sql-mariadb-106 ports: - - 60103:3306 + - 60106:3306 - mariadb-102: + mariadb-105: <<: *mariadb build: context: ./test/docker/mariadb args: - IMAGE: mariadb:10.2.36 - container_name: org-sql-mariadb-102 + IMAGE: mariadb:10.5.25 + container_name: org-sql-mariadb-105 ports: - - 60102:3306 - + - 60105:3306 + ## mysql mysql-latest: @@ -102,19 +92,20 @@ services: build: context: ./test/docker/mariadb args: - IMAGE: mysql:8.0.23 + IMAGE: mysql:8.4.2 container_name: org-sql-mysql-latest ports: - - 60280:3306 + - 60284:3306 - mysql57: + mysql80: + <<: *mariadb build: context: ./test/docker/mariadb args: - IMAGE: mysql:5.7.33 - container_name: org-sql-mysql-57 + IMAGE: mysql:8.0.39 + container_name: org-sql-mysql-80 ports: - - 60257:3306 + - 60280:3306 ## sqlserver @@ -124,7 +115,17 @@ services: restart: 'no' environment: - ACCEPT_EULA=Y - - SA_PASSWORD=org_sql222@@@ + - SA_PASSWORD=SFDwIcGvZdx&9g1f4Uy + ports: + - 60322:1433 + + sqlserver-2019: + <<: *sqlserver + container_name: org-sql-sqlserver-2019 + build: + context: ./test/docker/sql-server + args: + IMAGE: mcr.microsoft.com/mssql/server:2019-CU27-ubuntu-20.04 ports: - 60319:1433 @@ -134,6 +135,6 @@ services: build: context: ./test/docker/sql-server args: - IMAGE: mcr.microsoft.com/mssql/server:2017-CU22-ubuntu-16.04 + IMAGE: mcr.microsoft.com/mssql/server:2017-CU29-ubuntu-16.04 ports: - 60317:1433 diff --git a/env-29.3.yml b/env-29.3.yml new file mode 100644 index 0000000..7aebbc1 --- /dev/null +++ b/env-29.3.yml @@ -0,0 +1,137 @@ +name: org-sql-29.3 +channels: + - conda-forge +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - adwaita-icon-theme=46.2=unix_0 + - at-spi2-atk=2.38.0=h0630a04_3 + - at-spi2-core=2.40.3=h0630a04_0 + - atk-1.0=2.38.0=h04ea711_2 + - bzip2=1.0.8=h4bc722e_7 + - ca-certificates=2024.7.4=hbcca054_0 + - cairo=1.18.0=hbb29018_2 + - dbus=1.13.6=h5008d03_3 + - emacs=29.3=hc93ec10_0 + - epoxy=1.5.10=h166bdaf_1 + - expat=2.6.2=h59595ed_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - freetype=2.12.1=h267a509_2 + - fribidi=1.0.10=h36c2ea0_0 + - gdk-pixbuf=2.42.12=hb9ae30d_0 + - gettext=0.22.5=he02047a_3 + - gettext-tools=0.22.5=he02047a_3 + - giflib=5.2.2=hd590300_0 + - glib=2.80.3=h315aac3_2 + - glib-tools=2.80.3=h8fdd7da_2 + - gmp=6.3.0=hac33072_2 + - gnutls=3.7.9=hb077bed_0 + - go-sqlcmd=1.8.0=ha8f183a_0 + - graphite2=1.3.13=h59595ed_1003 + - gtk3=3.24.43=h0359ba6_0 + - harfbuzz=8.5.0=hfac3d4d_0 + - hicolor-icon-theme=0.17=ha770c72_2 + - icu=73.2=h59595ed_0 + - jansson=2.14=h0b41bf4_1 + - keyutils=1.6.1=h166bdaf_0 + - krb5=1.21.3=h659f571_0 + - ld_impl_linux-64=2.40=hf3520f5_7 + - lerc=4.0.0=h27087fc_0 + - libasprintf=0.22.5=he8f35ee_3 + - libasprintf-devel=0.22.5=he8f35ee_3 + - libcups=2.3.3=h4637d8d_4 + - libdeflate=1.21=h4bc722e_0 + - libedit=3.1.20191231=he28a2e2_2 + - libexpat=2.6.2=h59595ed_0 + - libffi=3.4.2=h7f98852_5 + - libgcc-ng=14.1.0=h77fa898_0 + - libgettextpo=0.22.5=he02047a_3 + - libgettextpo-devel=0.22.5=he02047a_3 + - libglib=2.80.3=h315aac3_2 + - libgomp=14.1.0=h77fa898_0 + - libiconv=1.17=hd590300_2 + - libidn2=2.3.7=hd590300_0 + - libjpeg-turbo=3.0.0=hd590300_1 + - libnsl=2.0.1=hd590300_0 + - libpng=1.6.43=h2797004_0 + - libpq=16.4=h482b261_0 + - librsvg=2.58.2=hf0cb8fb_0 + - libsqlite=3.46.0=hde9e2c9_0 + - libstdcxx-ng=14.1.0=hc0a3c3a_0 + - libtasn1=4.19.0=h166bdaf_0 + - libtiff=4.6.0=h46a8edc_4 + - libtree-sitter=0.22.6=h4ab18f5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.16=hd590300_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.7.0=h2c5496b_1 + - libxml2=2.12.7=h4c95cb1_3 + - libzlib=1.3.1=h4ab18f5_1 + - mysql-common=8.3.0=h70512c7_5 + - mysql-libs=8.3.0=ha479ceb_5 + - mysqlclient=2.2.4=py312h30efb56_1 + - ncurses=6.5=h59595ed_0 + - nettle=3.9.1=h7ab15ed_0 + - openssl=3.3.1=h4bc722e_2 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=24.1=pyhd8ed1ab_0 + - pango=1.54.0=h84a9a3c_0 + - pcre2=10.44=hba22ea6_2 + - pip=24.2=pyhd8ed1ab_0 + - pixman=0.43.2=h59595ed_0 + - postgresql=16.4=ha8faf9a_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - python=3.12.5=h2ad013b_0_cpython + - python_abi=3.12=5_cp312 + - readline=8.2=h8228510_1 + - setuptools=72.1.0=pyhd8ed1ab_0 + - sqlite=3.46.0=h6d4b2fc_0 + - tk=8.6.13=noxft_h4845f30_101 + - tzcode=2024a=h3f72095_0 + - tzdata=2024a=h0c530f3_0 + - wayland=1.23.0=h5291e77_0 + - wheel=0.44.0=pyhd8ed1ab_0 + - xkeyboard-config=2.42=h4ab18f5_0 + - xorg-compositeproto=0.4.2=h7f98852_1001 + - xorg-damageproto=1.2.1=h7f98852_1002 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-inputproto=2.3.2=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=hb711507_1 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxaw=1.0.14=h7f98852_1 + - xorg-libxcomposite=0.4.6=h0b41bf4_1 + - xorg-libxcursor=1.2.0=h0b41bf4_1 + - xorg-libxdamage=1.1.5=h7f98852_1 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxft=2.3.8=hf69aa0a_0 + - xorg-libxi=1.7.10=h4bc722e_1 + - xorg-libxinerama=1.1.5=h27087fc_0 + - xorg-libxmu=1.1.3=h4ab18f5_1 + - xorg-libxpm=3.5.17=hd590300_0 + - xorg-libxrandr=1.5.2=h7f98852_1 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-libxt=1.3.0=hd590300_1 + - xorg-libxtst=1.2.5=h4bc722e_0 + - xorg-randrproto=1.5.0=h7f98852_1001 + - xorg-recordproto=1.14.2=h7f98852_1002 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-util-macros=1.19.3=h7f98852_0 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xineramaproto=1.2.1=h7f98852_1001 + - xorg-xproto=7.0.31=h7f98852_1007 + - xz=5.2.6=h166bdaf_0 + - zlib=1.3.1=h4ab18f5_1 + - zstd=1.5.6=ha6fb4c9_0 diff --git a/init.el b/init.el new file mode 100644 index 0000000..68ccdd8 --- /dev/null +++ b/init.el @@ -0,0 +1,51 @@ +(defun fix-null-term (s) + "Fix string S with extra wonky null terminators. + +For whatever reason this affects certain strings in the conda +package for Emacs. These look like `blabla\0\0\0\0\0\0\0`." + (declare (pure t) (side-effect-free t)) + (save-match-data + (if (string-match "\0+" s) + (replace-match "" t t s) + s))) + +;; HACK stuff won't install unless this string is fixed +(if (< (round (string-to-number emacs-version)) 29) + (setq Info-default-directory-list + (cons + (fix-null-term (car Info-default-directory-list)) + (cdr Info-default-directory-list))) + (setq configure-info-directory (fix-null-term configure-info-directory))) + +(setq package-enable-at-startup nil) + +(setq user-emacs-directory + (file-name-concat (expand-file-name ".emacs") emacs-version)) + +(defvar bootstrap-version) + +(let ((bootstrap-file + (file-name-concat + user-emacs-directory + "straight/repos/straight.el/bootstrap.el")) + (bootstrap-version 7)) + (unless (file-exists-p bootstrap-file) + (with-current-buffer + (url-retrieve-synchronously + "https://raw.githubusercontent.com/radian-software/straight.el/develop/install.el" + 'silent 'inhibit-cookies) + (goto-char (point-max)) + (eval-print-last-sexp))) + (load bootstrap-file nil 'nomessage)) + +(straight-use-package 's) +(straight-use-package 'f) +(straight-use-package 'dash) +(straight-use-package 'buttercup) +(straight-use-package 'org) +(straight-use-package 'org-ml) + +(defun compile-target () + "Compile org-ml." + (byte-compile-file "org-ml-macs.el") + (byte-compile-file "org-ml.el")) diff --git a/org-sql.el b/org-sql.el index f2a9377..1e49083 100644 --- a/org-sql.el +++ b/org-sql.el @@ -5,8 +5,8 @@ ;; Author: Nathan Dwarshuis ;; Keywords: org-mode, data ;; Homepage: https://github.com/ndwarshuis/org-sql -;; Package-Requires: ((emacs "27.1") (s "1.12") (f "0.20.0") (dash "2.17") (org-ml "5.6.1")) -;; Version: 3.0.4 +;; Package-Requires: ((emacs "27.1") (s "1.13") (f "0.20.0") (dash "2.19.1") (org-ml "5.8.8")) +;; Version: 4.0.0 ;; This program is free software; you can redistribute it and/or modify ;; it under the terms of the GNU General Public License as published by @@ -324,7 +324,7 @@ to store them. This is in addition to any properties specifified by :parent-keys (:timestamp_id) :on-delete cascade :cardinality one-or-none-to-one))) - + (timestamp_repeaters (desc "Each row stores the repeater component for a timestamp." "If the repeater also has a habit appended to it, this will" @@ -447,7 +447,7 @@ to store them. This is in addition to any properties specifified by :parent-keys (:headline_id) :on-delete cascade :cardinality many-or-none-to-one))) - + (clocks (desc "Each row stores one clock entry.") (columns @@ -875,7 +875,7 @@ This works analogously to `org-sql-post-init-hooks'." (defcustom org-sql-excluded-properties nil "List of properties to exclude from the database. -To exclude all set to 'all' instead of a list of strings." +To exclude all set to `all' instead of a list of strings." :type '(choice (const "Ignore All" all) (repeat :tag "List of properties to ignore" string)) @@ -888,7 +888,7 @@ To exclude all set to 'all' instead of a list of strings." (defcustom org-sql-excluded-tags nil "List of tags to exclude when building the tags table. -To exclude all set to 'all' instead of a list of strings." +To exclude all set to `all' instead of a list of strings." :type '(choice (const "Ignore All" all) (repeat :tag "List of tags to ignore" string)) @@ -899,7 +899,7 @@ To exclude all set to 'all' instead of a list of strings." Each member should be a string and one of `org-link-types' or \"file\", \"coderef\", \"custom-id\", \"fuzzy\", or \"id\". See org-element API documentation or`org-element-link-parser' for details. -To exclude all set to 'all' instead of a list of strings." +To exclude all set to `all' instead of a list of strings." :type '(choice (set :tag "List of types to ignore" (const :tag "File paths" "file") @@ -923,8 +923,8 @@ exclude none set to nil." (defcustom org-sql-excluded-contents-timestamp-types nil "List of timestamp types to exclude from headline content sections. -List members can be the symbols 'active', 'active-range', 'inactive', -or 'inactive-range'. To exclude none set to nil." +List members can be the symbols `active', `active-range', `inactive', +or `inactive-range'. To exclude none set to nil." :type '(set :tag "List of types to include" (const :tag "Active Timestamps" active) (const :tag "Active Timestamp Ranges" active-range) @@ -936,7 +936,7 @@ or 'inactive-range'. To exclude none set to nil." (defcustom org-sql-excluded-logbook-types nil "List of logbook entry types to exclude from the database. List members are any of the keys from `org-log-note-headings' with the -exception of 'clock-out' as these are treated as clock-notes (see +exception of `clock-out' as these are treated as clock-notes (see `org-sql-exclude-clock-notes'). To include none set to nil." :type '(set :tag "List of types to include" (const :tag "Clocks" clock) @@ -1302,7 +1302,8 @@ values of one row from OUT." ((mysql sqlserver) "\n") ((postgres sqlite) org-sql--row-sep)))) (->> (org-sql--case-mode config - ((mysql postgres sqlserver) (s-chomp out)) + ((mysql postgres) (s-chomp out)) + (sqlserver (s-trim-right out)) (sqlite (s-chop-suffix rsep out))) (s-split rsep) (--map (s-split fsep it)))))) @@ -1373,7 +1374,7 @@ plists (where the keys are the column names for the rows)." ;; TODO add this to org-ml (defun org-sql--build-org-data (preamble headlines) - "Return a new 'org-data' node type. + "Return a new `org-data' node type. PREAMBLE is a section node representing the \"preamble\" (the text above the first headline, if any) or nil, and HEADLINES is a list of headline nodes." @@ -1775,8 +1776,8 @@ CONTENTS is a list corresponding to that returned by (org-sql--insert-alist-add-timestamp-repeater it timestamp) (org-sql--insert-alist-add-timestamp-warning it timestamp))))) -(defun org-sql--insert-alist-add-headline-timestamps (acc contents) - "Add row for each timestamp in the current headline to ACC. +(defun org-sql--insert-alist-add-headline-timestamps-from-contents (acc contents) + "Add row for each timestamp in the current headline's contents to ACC. CONTENTS is a list corresponding to that returned by `org-ml-headline-get-supercontents'." (if (eq org-sql-excluded-contents-timestamp-types 'all) acc @@ -1791,6 +1792,16 @@ CONTENTS is a list corresponding to that returned by acc timestamps)) acc))) +(defun org-sql--insert-alist-add-headline-timestamps-from-title (acc title) + "Add row for each timestamp in the current headline's title to ACC. +TITLE is pre-parsed to contain any timestamps from headline title." + (if (eq org-sql-excluded-contents-timestamp-types 'all) acc + (-if-let (timestamps (--filter (org-ml-is-type 'timestamp it) title)) + (--reduce-from (->> (org-sql--insert-alist-add-timestamp acc it) + (org-sql--acc-incr :timestamp-id)) + acc timestamps) + acc))) + (defun org-sql--insert-alist-add-headline-planning (acc hstate) "Add row for each planning timestamp in the current headline to ACC. HSTATE is a plist as returned by `org-sql--to-hstate'." @@ -1848,7 +1859,8 @@ INDEX is the order of the headline relative to its neighbors. (`(,_ 0) '(nil fraction)) (`(,n ,d) `(,(/ (* 1.0 n) d) fraction)) (`(,p) `(,p percent)))) - (contents (org-ml-supercontents-get-contents supercontents))) + (contents (org-ml-supercontents-get-contents supercontents)) + (title (org-ml-get-property :title headline))) (--> (org-sql--insert-alist-add acc headlines :headline_id (org-sql--acc-get :headline-id acc) :outline_hash outline-hash @@ -1869,7 +1881,8 @@ INDEX is the order of the headline relative to its neighbors. (org-sql--insert-alist-add-headline-planning it hstate) (org-sql--insert-alist-add-headline-tags it hstate) (org-sql--insert-alist-add-headline-properties it hstate) - (org-sql--insert-alist-add-headline-timestamps it contents) + (org-sql--insert-alist-add-headline-timestamps-from-contents it contents) + (org-sql--insert-alist-add-headline-timestamps-from-title it title) (org-sql--insert-alist-add-headline-links it contents) (org-sql--insert-alist-add-headline-logbook-clocks it hstate logbook) (org-sql--insert-alist-add-headline-logbook-items it hstate logbook) @@ -2784,7 +2797,6 @@ MySQL configs will additionally substitute tabs with "Return a SQL query that yields all headlines in the database. CONFIG is a list like `org-sql-db-config'." (let* ((hl-tbl (org-sql--format-table-name config "headlines")) - (ol-tbl (org-sql--format-table-name config "outlines")) (recursive-paths (org-sql--case-mode config ((mysql postgres sqlite) "RECURSIVE paths") (sqlserver "paths"))) @@ -2845,7 +2857,7 @@ CONFIG is a list like `org-sql-db-config'." "ORDER BY h.outline_hash, p.ipath;") ;; "LIMIT 50;") (s-join " ")))) - (format fmt cte columns hl-tbl ol-tbl))) + (format fmt cte columns hl-tbl))) (defun org-sql--compile-deserializer* (config type) "Return a function that deserializes TYPE. @@ -2916,7 +2928,7 @@ FIRST-FORM must return a cons cell like (RC . OUT) where RC is the return code and OUT is the output string (stdout and/or stderr). SUCCESS-FORM will be run if RC is 0 and ERROR-FORM will be run otherwise. In either case, OUT will be bound to the symbol -'it-out'." +`it-out'." (declare (indent 1)) (let ((r (make-symbol "rc"))) `(-let (((,r . it-out) ,first-form)) @@ -2924,7 +2936,7 @@ be run otherwise. In either case, OUT will be bound to the symbol (defmacro org-sql--on-success* (first-form &rest success-forms) "Run SUCCESS-FORMS on successful exit code. -This is like `org-sql--on-success' but with '(error it-out)' +This is like `org-sql--on-success' but with `(error it-out)' supplied for ERROR-FORM. FIRST-FORM has the same meaning." (declare (indent 1)) `(org-sql--on-success ,first-form (progn ,@success-forms) (error it-out))) @@ -3026,7 +3038,7 @@ Return a cons cell like (RETURNCODE . OUTPUT)." (with-temp-buffer (let ((rc (apply #'call-process path file (current-buffer) nil args))) (cons rc (buffer-string))))) - + ;;; hashpathpair -> outline-config (defun org-sql--hashpathpair-get-outline-config (outline-hash file-paths) @@ -3630,7 +3642,7 @@ connection. This will return a list of lists like (PATH . TREE) where PATH is the path to a tracked file and TREE is the org-tree for that file represented as an org-element node tree (specifically an -'org-data' node). Passing TREE to `org-ml-to-string' or +`org-data' node). Passing TREE to `org-ml-to-string' or `org-element-interpret-data' will produce the string representation of the file contents. diff --git a/test/docker/mariadb/Dockerfile b/test/docker/mariadb/Dockerfile index a215e7b..816bfab 100644 --- a/test/docker/mariadb/Dockerfile +++ b/test/docker/mariadb/Dockerfile @@ -1,4 +1,4 @@ -ARG IMAGE=mariadb:10.5.8 +ARG IMAGE=mariadb:10.11.4 FROM $IMAGE COPY ./init/* /docker-entrypoint-initdb.d/ diff --git a/test/docker/postgres/Dockerfile b/test/docker/postgres/Dockerfile index ef9db6d..548ec55 100644 --- a/test/docker/postgres/Dockerfile +++ b/test/docker/postgres/Dockerfile @@ -1,4 +1,4 @@ -ARG IMAGE=postgres:13.2-alpine +ARG IMAGE=postgres:16.4-alpine FROM $IMAGE COPY ./init/* /docker-entrypoint-initdb.d/ diff --git a/test/docker/postgres/init/org_sql.sql b/test/docker/postgres/init/org_sql.sql index 03186cc..9b8137b 100644 --- a/test/docker/postgres/init/org_sql.sql +++ b/test/docker/postgres/init/org_sql.sql @@ -7,3 +7,4 @@ CREATE ROLE org_sql WITH LOGIN PASSWORD 'org_sql'; -- create alt schema to test CREATE SCHEMA nonpublic AUTHORIZATION org_sql; GRANT ALL ON SCHEMA nonpublic TO org_sql; +GRANT ALL ON SCHEMA public TO org_sql; diff --git a/test/docker/sql-server/Dockerfile b/test/docker/sql-server/Dockerfile index 3bd5edd..81377d1 100644 --- a/test/docker/sql-server/Dockerfile +++ b/test/docker/sql-server/Dockerfile @@ -1,4 +1,4 @@ -ARG IMAGE=mcr.microsoft.com/mssql/server:2019-CU8-ubuntu-16.04 +ARG IMAGE=mcr.microsoft.com/mssql/server:2022-CU11-ubuntu-22.04 FROM $IMAGE ## Start the sql-server docker image with an init script to set up the test db diff --git a/test/docker/sql-server/init-db.sh b/test/docker/sql-server/init-db.sh index 1bc2314..ff4a182 100644 --- a/test/docker/sql-server/init-db.sh +++ b/test/docker/sql-server/init-db.sh @@ -2,7 +2,7 @@ #do this in a loop because the timing for when the SQL instance is ready is indeterminate for i in {1..50}; do - /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P org_sql222@@@ \ + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "SFDwIcGvZdx&9g1f4Uy" \ -d master -i /usr/src/app/setup.sql if [ $? -eq 0 ] then diff --git a/test/docker/sql-server/setup.sql b/test/docker/sql-server/setup.sql index 029e3b1..ee5548b 100644 --- a/test/docker/sql-server/setup.sql +++ b/test/docker/sql-server/setup.sql @@ -1,6 +1,6 @@ CREATE DATABASE org_sql; GO -CREATE LOGIN org_sql WITH PASSWORD = 'org_sql333###'; +CREATE LOGIN org_sql WITH PASSWORD = 'o%4XlS14tPO!J@q@16v'; GO USE org_sql; GO diff --git a/test/org-sql-test-stateful.el b/test/org-sql-test-stateful.el index 7110bc2..614fa4f 100644 --- a/test/org-sql-test-stateful.el +++ b/test/org-sql-test-stateful.el @@ -655,7 +655,7 @@ :server (format "tcp:localhost,%s" port) :args '("-C") ;; trust server cert :username "org_sql" - :password "org_sql333###") + :password "o%4XlS14tPO!J@q@16v") (append key-vals) (mk-io-spec "SQL-Server" 'sqlserver version alt-title)))) (let* ((sqlite (list "SQLite" @@ -663,34 +663,38 @@ "org-sql-test.db")))) (postgres (append + (mk-postgres 16 60016) + (mk-postgres 16 60016 "Non-Default Schema" '(:schema "nonpublic")) + (mk-postgres 16 60016 "Unlogged tables" '(:unlogged t)) + (mk-postgres 15 60015) + (mk-postgres 14 60014) (mk-postgres 13 60013) - (mk-postgres 13 60013 "Non-Default Schema" '(:schema "nonpublic")) - (mk-postgres 13 60013 "Unlogged tables" '(:unlogged t)) - (mk-postgres 12 60012) - (mk-postgres 11 60011) - (mk-postgres 10 60010) - (mk-postgres 9 60009))) + )) (mariadb (append - (mk-mysql "MariaDB" 10.5 60105) - (mk-mysql "MariaDB" 10.4 60104) - (mk-mysql "MariaDB" 10.3 60103) - (mk-mysql "MariaDB" 10.2 60102))) + (mk-mysql "MariaDB" 11.4 60114) + (mk-mysql "MariaDB" 10.11 60111) + (mk-mysql "MariaDB" 10.6 60106) + (mk-mysql "MariaDB" 10.5 60105))) (mysql (append - (mk-mysql "MySQL" 8.0 60280) - ;; (mk-mysql "MySQL" 5.7 60257) - )) + (mk-mysql "MySQL" 8.4 60284) + (mk-mysql "MySQL" 8.0 60280))) (sqlserver (append + (mk-sqlserver 2022 60322 nil '(:schema "nondbo")) (mk-sqlserver 2019 60319 nil '(:schema "nondbo")) - (mk-sqlserver 2017 60317 nil '(:schema "nondbo"))))) + (mk-sqlserver 2017 60317 nil '(:schema "nondbo")) + ))) (eval `(describe-io-specs ,@sqlite ,@postgres ,@mariadb ,@mysql - ,@sqlserver)))) + ,@sqlserver + ) + t))) + ;;; org-sql-test-stateful ends here diff --git a/test/org-sql-test-stateless.el b/test/org-sql-test-stateless.el index 434f6c2..5745085 100644 --- a/test/org-sql-test-stateless.el +++ b/test/org-sql-test-stateless.el @@ -486,7 +486,7 @@ list then join the cdr of IN with newlines." (it "fancy" (expect-sql (list "stuff at the top" - "* TODO [#A] COMMENT another headline [1/2]" + "* TODO [#A] COMMENT another headline <2024-08-12 Mon> [1/2]" ":PROPERTIES:" ":Effort: 0:30" ":END:" @@ -494,10 +494,12 @@ list then join the cdr of IN with newlines." `((outlines (,@testing-outlines* "stuff at the top\n")) ,testing-file_metadata (headlines - (1 ,testing-hash "another headline [1/2]" 1 0 "TODO" 30 "A" fraction 0.5 + (1 ,testing-hash "another headline <2024-08-12 Mon> [1/2]" 1 0 "TODO" 30 "A" fraction 0.5 0 1 "this /should/ appear\n")) (headline_closures - (1 1 0))))) + (1 1 0)) + (timestamps (1 1 "<2024-08-12 Mon>" 1 + ,(org-ts-to-unixtime "<2024-08-12 Mon>") nil 0 nil))))) (expect-sql-tbls-multi (outlines file_metadata headlines headline_closures) (list "* headline" @@ -517,7 +519,7 @@ list then join the cdr of IN with newlines." ,testing-file_metadata (headlines (1 ,testing-hash "headline" 1 0 nil nil nil nil nil 0 0 nil)) (headline_closures (1 1 0))) - + "nested (no predicate)" nil `((outlines (,@testing-outlines* nil)) @@ -702,7 +704,7 @@ list then join the cdr of IN with newlines." "** child" ts) `((timestamps (1 2 ,ts 1 ,(org-ts-to-unixtime ts) nil 0 nil)))))) - + (it "content (ranged)" (let* ((ts0 "<2112-01-01 Thu>") (ts1 "<2112-01-02 Fri>") @@ -734,7 +736,7 @@ list then join the cdr of IN with newlines." "** child" "https://example.com") `((links (1 2 "//example.com" nil "https"))))) - + (it "with description" (expect-sql-tbls (links) (list "* parent" "[[https://example.org][relevant]]") @@ -998,7 +1000,7 @@ list then join the cdr of IN with newlines." ":END:") `((clocks (1 1 ,(org-ts-to-unixtime ts0) ,(org-ts-to-unixtime ts1) nil)) - (logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) + (logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) (it "non-note + clock + clock note" (let* ((org-log-note-clock-out t) @@ -1184,7 +1186,7 @@ list then join the cdr of IN with newlines." :postgres "'foo'" :sqlite "'foo'" :sqlserver "'foo'")) - + (it "newlines" (expect-formatter 'text "foo\nbar" :mysql "'foo\nbar'" @@ -1376,4 +1378,3 @@ list then join the cdr of IN with newlines." (files-to-delete ("789" . "/foo0.org"))))))) ;;; org-sql-test-stateless.el ends here -