From fb7a5f02ebb9b0939fe4de2799dfe8843155eb05 Mon Sep 17 00:00:00 2001 From: Hojjat Jafarpour Date: Thu, 25 Jan 2018 17:09:46 -0800 Subject: [PATCH 1/7] Cos integration for KSQL. --- debian/Makefile | 152 +++++++++++++++++++++ debian/changelog | 5 + debian/compat | 1 + debian/confluent-kafka-rest.postinst | 46 +++++++ debian/confluent-kafka-rest.service | 13 ++ debian/confluent-kafka-rest.spec.in | 101 ++++++++++++++ debian/control | 13 ++ debian/copyright | 27 ++++ debian/gbp.conf | 5 + debian/patches/series | 1 + debian/patches/standard_build_layout.patch | 14 ++ debian/rules | 10 ++ debian/source/format | 1 + 13 files changed, 389 insertions(+) create mode 100644 debian/Makefile create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/confluent-kafka-rest.postinst create mode 100644 debian/confluent-kafka-rest.service create mode 100644 debian/confluent-kafka-rest.spec.in create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/gbp.conf create mode 100644 debian/patches/series create mode 100644 debian/patches/standard_build_layout.patch create mode 100755 debian/rules create mode 100644 debian/source/format diff --git a/debian/Makefile b/debian/Makefile new file mode 100644 index 000000000000..49207dbdbb2a --- /dev/null +++ b/debian/Makefile @@ -0,0 +1,152 @@ +# Dependencies you'll probably need to install to compile this: make, curl, git, +# zip, unzip, patch, java7-jdk | openjdk-7-jdk, maven. + +SHELL=/bin/bash + +ifndef VERSION +# Note that this is sensitive to this package's version being the first +# tag in the pom.xml +VERSION=$(shell grep version pom.xml | head -n 1 | awk -F'>|<' '{ print $$3 }') +endif + +export PACKAGE_TITLE=ksql +export FULL_PACKAGE_TITLE=confluent-ksql +export PACKAGE_NAME=$(FULL_PACKAGE_TITLE)-$(VERSION) + +# Defaults that are likely to vary by platform. These are cleanly separated so +# it should be easy to maintain altered values on platform-specific branches +# when the values aren't overridden by the script invoking the Makefile + +# Whether we should apply patches. This only makes sense for alternate packaging +# systems that know how to apply patches themselves, e.g. Debian. +APPLY_PATCHES?=yes + +# DESTDIR may be overriden by e.g. debian packaging +ifeq ($(DESTDIR),) +DESTDIR=$(CURDIR)/BUILD/ +endif + +ifeq ($(PACKAGE_TYPE),archive) +PREFIX=$(PACKAGE_NAME) +SYSCONFDIR=$(PREFIX)/etc/$(PACKAGE_TITLE) +SYSTEMDDIR=$(PREFIX)/lib/systemd/system +else +PREFIX=/usr +SYSCONFDIR=/etc/$(PACKAGE_TITLE) +SYSTEMDDIR=/lib/systemd/system +endif + +# Whether we should run tests during the build. +SKIP_TESTS?=yes + +all: install + + +archive: install + rm -f $(CURDIR)/$(PACKAGE_NAME).tar.gz && cd $(DESTDIR) && tar -czf $(CURDIR)/$(PACKAGE_NAME).tar.gz $(PREFIX) + rm -f $(CURDIR)/$(PACKAGE_NAME).zip && cd $(DESTDIR) && zip -r $(CURDIR)/$(PACKAGE_NAME).zip $(PREFIX) + +apply-patches: $(wildcard patches/*) +ifeq ($(APPLY_PATCHES),yes) + git reset --hard HEAD + cat patches/series | xargs -iPATCH bash -c 'patch -p1 < patches/PATCH' +endif + +build: apply-patches +ifeq ($(SKIP_TESTS),yes) + mvn -DskipTests=true install +else + mvn install +endif + +BINPATH=$(PREFIX)/bin +LIBPATH=$(PREFIX)/share/java/$(PACKAGE_TITLE) +DOCPATH=$(PREFIX)/share/doc/$(PACKAGE_TITLE) + +INSTALL=install -D -m 644 +INSTALL_X=install -D -m 755 + +install: build + # Safety precatuion to avoid removing of root dir when DESTDIR or PREFIX is not set, for wathever reason + if [[ $(DESTDIR)$(PREFIX) != /tmp/confluent/* ]]; then echo "DESTDIR=$(DESTDIR) or PREFIX=$(PREFIX) is weird" ; exit 1 ; fi + rm -rf $(DESTDIR)$(PREFIX) + mkdir -p $(DESTDIR)$(PREFIX) + mkdir -p $(DESTDIR)$(BINPATH) + mkdir -p $(DESTDIR)$(LIBPATH) + mkdir -p $(DESTDIR)$(SYSCONFDIR) + for svc in debian/*.service ; do \ + $(INSTALL) -o root -g root -D $$svc $(DESTDIR)$(SYSTEMDDIR)/$$(basename $$svc) ; \ + done + pushd "ksql/target/ksql-$(VERSION)-package" ; \ + find bin/ -type f | grep -v README[.]rpm | xargs -I XXX $(INSTALL_X) -o root -g root XXX $(DESTDIR)$(PREFIX)/XXX ;\ + find share/ -type f | grep -v README[.]rpm | xargs -I XXX $(INSTALL) -o root -g root XXX $(DESTDIR)$(PREFIX)/XXX ; \ + pushd etc/ksql/ ; \ + find . -type f | grep -v README[.]rpm | xargs -I XXX $(INSTALL) -o root -g root XXX $(DESTDIR)$(SYSCONFDIR)/XXX + + +clean: + rm -rf $(CURDIR)/$(PACKAGE_NAME)* + rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)*rpm + rm -rf RPM_BUILDING + +distclean: clean + git reset --hard HEAD + git status --ignored --porcelain | cut -d ' ' -f 2 | xargs rm -rf + +test: + +.PHONY: clean install + + + +RPM_VERSION=$(shell echo $(VERSION) | sed -e 's/-alpha[0-9]*//' -e 's/-beta[0-9]*//' -e 's/-rc[0-9]*//' -e 's/-SNAPSHOT//') +# Get any -alpha, -beta, -rc piece that we need to put into the Release part of +# the version since RPM versions don't support non-numeric +# characters. Ultimately, for something like 0.8.2-beta, we want to end up with +# Version=0.8.2 Release=0.X.beta +# where X is the RPM release # of 0.8.2-beta (the prefix 0. forces this to be +# considered earlier than any 0.8.2 final releases since those will start with +# Version=0.8.2 Release=1) +RPM_RELEASE_POSTFIX=$(subst -,,$(subst $(RPM_VERSION),,$(VERSION))) +ifneq ($(RPM_RELEASE_POSTFIX),) + RPM_RELEASE_POSTFIX_UNDERSCORE=_$(RPM_RELEASE_POSTFIX) + RPM_RELEASE_ID=0.$(REVISION).$(RPM_RELEASE_POSTFIX) +else + RPM_RELEASE_ID=$(REVISION) +endif + + + +rpm: RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz + echo "Building the rpm" + rpmbuild --define="_topdir `pwd`/RPM_BUILDING" -tb $< + find RPM_BUILDING/{,S}RPMS/ -type f | xargs -n1 -iXXX mv XXX . + echo + echo "=================================================" + echo "The rpms have been created and can be found here:" + @ls -laF $(FULL_PACKAGE_TITLE)*rpm + echo "=================================================" + +# Unfortunately, because of version naming issues and the way rpmbuild expects +# the paths in the tar file to be named, we need to rearchive the package. So +# instead of depending on archive, this target just uses the unarchived, +# installed version to generate a new archive. Note that we always regenerate +# the symlink because the RPM_VERSION doesn't include all the version info -- it +# can leave of things like -beta, -rc1, etc. +RPM_BUILDING/SOURCES/$(FULL_PACKAGE_TITLE)-$(RPM_VERSION).tar.gz: rpm-build-area install debian/$(FULL_PACKAGE_TITLE).spec.in RELEASE_$(RPM_VERSION)$(RPM_RELEASE_POSTFIX_UNDERSCORE) + rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + mkdir $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + cp -R $(DESTDIR)/* $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + sed "s@##RPMVERSION##@$(RPM_VERSION)@g; s@##RPMRELEASE##@$(RPM_RELEASE_ID)@g" < debian/$(FULL_PACKAGE_TITLE).spec.in > $(FULL_PACKAGE_TITLE)-$(RPM_VERSION)/$(FULL_PACKAGE_TITLE).spec + rm -f $@ && tar -czf $@ $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + rm -rf $(FULL_PACKAGE_TITLE)-$(RPM_VERSION) + +rpm-build-area: RPM_BUILDING/BUILD RPM_BUILDING/RPMS RPM_BUILDING/SOURCES RPM_BUILDING/SPECS RPM_BUILDING/SRPMS + +RPM_BUILDING/%: + mkdir -p $@ + +RELEASE_%: + echo 0 > $@ + +check: diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 000000000000..f2844ba8d9dd --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +confluent-ksql (0.1~SNAPSHOT-1) unstable; urgency=low + + * Initial release. + + -- Confluent Packaging Fri, 02 Dec 2015 22:09:36 +0000 diff --git a/debian/compat b/debian/compat new file mode 100644 index 000000000000..ec635144f600 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/confluent-kafka-rest.postinst b/debian/confluent-kafka-rest.postinst new file mode 100644 index 000000000000..7b1d5b6f9f22 --- /dev/null +++ b/debian/confluent-kafka-rest.postinst @@ -0,0 +1,46 @@ +#!/bin/sh + +set -e + +case "$1" in + configure) + _user=cp-ksql + _group=confluent + + (getent group $_group 2>&1 >/dev/null) || addgroup --quiet --system $_group + adduser --system --disabled-password --disabled-login --home /var/empty \ + --no-create-home --quiet --force-badname --ingroup $_group $_user + + _permwarn= + for dir in /var/log/confluent ; do + if [ $dir = /var/log/confluent ]; then + # Confluent log directory should be writable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+wx,o= $dir" + else + # Other dirs are only readable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+r,o= $dir" + fi + + if [ ! -d $dir ]; then + echo "Creating directory $dir with owner $_user:$_group" + mkdir -p $dir + eval $_perm + else + echo "Notice: Not creating existing directory $dir, ensure proper permissions for user $_user group $_group" + _permwarn="${_permwarn}${_perm}\n" + fi + done + + if [ -n "$_permwarn" ]; then + echo "Notice: If you are planning to use the provided systemd service units for" + echo "Notice: confluent-schema-registry, make sure that read-write permissions" + echo "Notice: for user $_user and group $_group are set up according to the" + echo "Notice: following commands:" + /bin/echo -e "$_permwarn" + fi + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/confluent-kafka-rest.service b/debian/confluent-kafka-rest.service new file mode 100644 index 000000000000..068dda1de0e3 --- /dev/null +++ b/debian/confluent-kafka-rest.service @@ -0,0 +1,13 @@ +[Unit] +Description=Streaming SQL engine for Apache Kafka +Documentation=http://docs.confluent.io/ +After=network.target confluent-kafka.target + +[Service] +Type=simple +User=cp-ksql +Group=confluent +Environment="LOG_DIR=/var/log/confluent/ksql" +ExecStart=/usr/bin/kafka-ksql-start /etc/kafka-ksql/kafka-ksql.properties +TimeoutStopSec=180 +Restart=no diff --git a/debian/confluent-kafka-rest.spec.in b/debian/confluent-kafka-rest.spec.in new file mode 100644 index 000000000000..a00618d126be --- /dev/null +++ b/debian/confluent-kafka-rest.spec.in @@ -0,0 +1,101 @@ +Summary: Streaming SQL engine for Kafka +Name: confluent-ksql +Version: ##RPMVERSION## +Release: ##RPMRELEASE## +Group: Applications/Internet +License: Apache (v2) +Source0: confluent-ksql-%{version}.tar.gz +URL: http://confluent.io +BuildRoot: %{_tmppath}/%{name}-%{version}-root +Vendor: Confluent, Inc. +Packager: Confluent Packaging +BuildArch: noarch + +Requires: confluent-rest-utils + +%description + +Streaming SQL engine for Apache Kafka. + +%define __jar_repack %{nil} + +%pre + +_user=cp-ksql +_group=confluent + +getent group $_group 2>&1 >/dev/null || groupadd -r $_group +getent passwd $_user 2>&1 >/dev/null || \ + useradd -r -g $_group --home-dir /tmp --no-create-home -s /sbin/nologin -c "Confluent KSQL" $_user + +_permwarn= +for dir in /var/log/confluent ; do + if [ $dir = /var/log/confluent ]; then + # Confluent log directory should be writable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+wx,o= $dir" + else + # Other dirs are only readable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+r,o= $dir" + fi + + if [ ! -d $dir ]; then + echo "Creating directory $dir with owner $_user:$_group" + mkdir -p $dir + eval $_perm + else + echo "Notice: Not creating existing directory $dir, ensure proper permissions for user $_user group $_group" + _permwarn="${_permwarn}${_perm}\n" + fi +done + +if [ -n "$_permwarn" ]; then + echo "Notice: If you are planning to use the provided systemd service units for" + echo "Notice: confluent-schema-registry, make sure that read-write permissions" + echo "Notice: for user $_user and group $_group are set up according to the" + echo "Notice: following commands:" + /bin/echo -e "$_permwarn" +fi + +%post + +%preun + +%postun + +%prep + +%setup + +%build + +%install + +# Clean out any previous builds not on slash +[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} + +%{__mkdir_p} %{buildroot} +%{__cp} -R * %{buildroot} +# The spec file gets included, get rid of it +%{__rm} %{buildroot}/confluent-kafka-rest.spec + +%{__mkdir_p} %{buildroot}/var/log/confluent + + +%files +%defattr(-,root,root) +/usr/bin/* +/usr/share/java/kafka-ksql +/lib/systemd/system/ + +%doc +/usr/share/doc/kafka-ksql + +%config(noreplace) /etc/kafka-ksql/* + +%clean +#used to cleanup things outside the build area and possibly inside. +[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} + +%changelog +* Fri Jan 2 2015 Confluent Packaging +- Initial import diff --git a/debian/control b/debian/control new file mode 100644 index 000000000000..24b8c5ed2e87 --- /dev/null +++ b/debian/control @@ -0,0 +1,13 @@ +Source: confluent-ksql +Section: misc +Priority: optional +Maintainer: Confluent Packaging +Build-Depends: debhelper (>= 9), dh-systemd, java8-jdk, javahelper (>= 0.40), + make, maven +Standards-Version: 3.9.3 +Homepage: http://confluent.io + +Package: confluent-ksql +Architecture: all +Depends: ${misc:Depends}, confluent-rest-utils, adduser +Description: Streaming SQL engine for Apache Kafka. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 000000000000..ad808733fdb0 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,27 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: ksql +Source: https://github.com/confluentinc/ksql + +Files: * +Copyright: 2014 Confluent, Inc. +License: Apache-2 + +Files: debian/* +Copyright: 2014 Confluent, Inc. +License: Apache-2 + +License: Apache-2 + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language overning permissions and limitations under + the License. + . + On Debian systems, the Apache 2.0 license can be found in + /usr/share/common-licenses/Apache-2.0. diff --git a/debian/gbp.conf b/debian/gbp.conf new file mode 100644 index 000000000000..ff98d871a32e --- /dev/null +++ b/debian/gbp.conf @@ -0,0 +1,5 @@ +[DEFAULT] +upstream-tree=tag +upstream-tag=%(version)s +debian-tag=debian-%(version)s +debian-branch=debian-%(version)s \ No newline at end of file diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 000000000000..16970d16ba64 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +standard_build_layout.patch diff --git a/debian/patches/standard_build_layout.patch b/debian/patches/standard_build_layout.patch new file mode 100644 index 000000000000..2d96aebd8216 --- /dev/null +++ b/debian/patches/standard_build_layout.patch @@ -0,0 +1,14 @@ +diff --git a/Makefile b/Makefile +new file mode 100644 +index 0000000..3f83b80 +--- /dev/null ++++ b/Makefile +@@ -0,0 +1,8 @@ ++clean: ++ ++distclean: ++ ++%: ++ cp debian/Makefile Makefile.debian ++ $(MAKE) -f Makefile.debian $@ ++ rm -f Makefile.debian diff --git a/debian/rules b/debian/rules new file mode 100755 index 000000000000..1a40dd2c882a --- /dev/null +++ b/debian/rules @@ -0,0 +1,10 @@ +#!/usr/bin/make -f + +%: + dh $@ --with javahelper,systemd + +override_dh_systemd_enable: + dh_systemd_enable --no-enable + +override_dh_systemd_start: + dh_systemd_start --no-start diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 000000000000..163aaf8d82b6 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) From c0dfac8a0c1e0c1008d1f6458373bd951712ec75 Mon Sep 17 00:00:00 2001 From: Hojjat Jafarpour Date: Thu, 25 Jan 2018 17:13:07 -0800 Subject: [PATCH 2/7] Changed java version in comments to 8. --- debian/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/Makefile b/debian/Makefile index 49207dbdbb2a..74a74a40cf97 100644 --- a/debian/Makefile +++ b/debian/Makefile @@ -1,5 +1,5 @@ # Dependencies you'll probably need to install to compile this: make, curl, git, -# zip, unzip, patch, java7-jdk | openjdk-7-jdk, maven. +# zip, unzip, patch, java8-jdk | openjdk-8-jdk, maven. SHELL=/bin/bash From 5f5981d692b245ffe6e1470a406cd76ad471ef2b Mon Sep 17 00:00:00 2001 From: Hojjat Jafarpour Date: Fri, 26 Jan 2018 09:28:54 -0800 Subject: [PATCH 3/7] Applied review feedback. Renamed the files for ksql. --- debian/confluent-kafka-rest.postinst | 46 ------------ debian/confluent-kafka-rest.service | 13 ---- debian/confluent-kafka-rest.spec.in | 101 --------------------------- 3 files changed, 160 deletions(-) delete mode 100644 debian/confluent-kafka-rest.postinst delete mode 100644 debian/confluent-kafka-rest.service delete mode 100644 debian/confluent-kafka-rest.spec.in diff --git a/debian/confluent-kafka-rest.postinst b/debian/confluent-kafka-rest.postinst deleted file mode 100644 index 7b1d5b6f9f22..000000000000 --- a/debian/confluent-kafka-rest.postinst +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh - -set -e - -case "$1" in - configure) - _user=cp-ksql - _group=confluent - - (getent group $_group 2>&1 >/dev/null) || addgroup --quiet --system $_group - adduser --system --disabled-password --disabled-login --home /var/empty \ - --no-create-home --quiet --force-badname --ingroup $_group $_user - - _permwarn= - for dir in /var/log/confluent ; do - if [ $dir = /var/log/confluent ]; then - # Confluent log directory should be writable by group - _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+wx,o= $dir" - else - # Other dirs are only readable by group - _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+r,o= $dir" - fi - - if [ ! -d $dir ]; then - echo "Creating directory $dir with owner $_user:$_group" - mkdir -p $dir - eval $_perm - else - echo "Notice: Not creating existing directory $dir, ensure proper permissions for user $_user group $_group" - _permwarn="${_permwarn}${_perm}\n" - fi - done - - if [ -n "$_permwarn" ]; then - echo "Notice: If you are planning to use the provided systemd service units for" - echo "Notice: confluent-schema-registry, make sure that read-write permissions" - echo "Notice: for user $_user and group $_group are set up according to the" - echo "Notice: following commands:" - /bin/echo -e "$_permwarn" - fi - ;; -esac - -#DEBHELPER# - -exit 0 diff --git a/debian/confluent-kafka-rest.service b/debian/confluent-kafka-rest.service deleted file mode 100644 index 068dda1de0e3..000000000000 --- a/debian/confluent-kafka-rest.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Streaming SQL engine for Apache Kafka -Documentation=http://docs.confluent.io/ -After=network.target confluent-kafka.target - -[Service] -Type=simple -User=cp-ksql -Group=confluent -Environment="LOG_DIR=/var/log/confluent/ksql" -ExecStart=/usr/bin/kafka-ksql-start /etc/kafka-ksql/kafka-ksql.properties -TimeoutStopSec=180 -Restart=no diff --git a/debian/confluent-kafka-rest.spec.in b/debian/confluent-kafka-rest.spec.in deleted file mode 100644 index a00618d126be..000000000000 --- a/debian/confluent-kafka-rest.spec.in +++ /dev/null @@ -1,101 +0,0 @@ -Summary: Streaming SQL engine for Kafka -Name: confluent-ksql -Version: ##RPMVERSION## -Release: ##RPMRELEASE## -Group: Applications/Internet -License: Apache (v2) -Source0: confluent-ksql-%{version}.tar.gz -URL: http://confluent.io -BuildRoot: %{_tmppath}/%{name}-%{version}-root -Vendor: Confluent, Inc. -Packager: Confluent Packaging -BuildArch: noarch - -Requires: confluent-rest-utils - -%description - -Streaming SQL engine for Apache Kafka. - -%define __jar_repack %{nil} - -%pre - -_user=cp-ksql -_group=confluent - -getent group $_group 2>&1 >/dev/null || groupadd -r $_group -getent passwd $_user 2>&1 >/dev/null || \ - useradd -r -g $_group --home-dir /tmp --no-create-home -s /sbin/nologin -c "Confluent KSQL" $_user - -_permwarn= -for dir in /var/log/confluent ; do - if [ $dir = /var/log/confluent ]; then - # Confluent log directory should be writable by group - _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+wx,o= $dir" - else - # Other dirs are only readable by group - _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+r,o= $dir" - fi - - if [ ! -d $dir ]; then - echo "Creating directory $dir with owner $_user:$_group" - mkdir -p $dir - eval $_perm - else - echo "Notice: Not creating existing directory $dir, ensure proper permissions for user $_user group $_group" - _permwarn="${_permwarn}${_perm}\n" - fi -done - -if [ -n "$_permwarn" ]; then - echo "Notice: If you are planning to use the provided systemd service units for" - echo "Notice: confluent-schema-registry, make sure that read-write permissions" - echo "Notice: for user $_user and group $_group are set up according to the" - echo "Notice: following commands:" - /bin/echo -e "$_permwarn" -fi - -%post - -%preun - -%postun - -%prep - -%setup - -%build - -%install - -# Clean out any previous builds not on slash -[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} - -%{__mkdir_p} %{buildroot} -%{__cp} -R * %{buildroot} -# The spec file gets included, get rid of it -%{__rm} %{buildroot}/confluent-kafka-rest.spec - -%{__mkdir_p} %{buildroot}/var/log/confluent - - -%files -%defattr(-,root,root) -/usr/bin/* -/usr/share/java/kafka-ksql -/lib/systemd/system/ - -%doc -/usr/share/doc/kafka-ksql - -%config(noreplace) /etc/kafka-ksql/* - -%clean -#used to cleanup things outside the build area and possibly inside. -[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} - -%changelog -* Fri Jan 2 2015 Confluent Packaging -- Initial import From 7b7056770f9049e29b0ceb89b37755281cbaddcc Mon Sep 17 00:00:00 2001 From: Hojjat Jafarpour Date: Fri, 26 Jan 2018 11:18:00 -0800 Subject: [PATCH 4/7] Added ksql confluent files. --- debian/confluent-ksql.postinst | 46 +++++++++++++++ debian/confluent-ksql.service | 13 +++++ debian/confluent-ksql.spec.in | 101 +++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 debian/confluent-ksql.postinst create mode 100644 debian/confluent-ksql.service create mode 100644 debian/confluent-ksql.spec.in diff --git a/debian/confluent-ksql.postinst b/debian/confluent-ksql.postinst new file mode 100644 index 000000000000..eb47d1648bc7 --- /dev/null +++ b/debian/confluent-ksql.postinst @@ -0,0 +1,46 @@ +#!/bin/sh + +set -e + +case "$1" in + configure) + _user=cp-ksql + _group=confluent + + (getent group $_group 2>&1 >/dev/null) || addgroup --quiet --system $_group + adduser --system --disabled-password --disabled-login --home /var/empty \ + --no-create-home --quiet --force-badname --ingroup $_group $_user + + _permwarn= + for dir in /var/log/confluent ; do + if [ $dir = /var/log/confluent ]; then + # Confluent log directory should be writable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+wx,o= $dir" + else + # Other dirs are only readable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+r,o= $dir" + fi + + if [ ! -d $dir ]; then + echo "Creating directory $dir with owner $_user:$_group" + mkdir -p $dir + eval $_perm + else + echo "Notice: Not creating existing directory $dir, ensure proper permissions for user $_user group $_group" + _permwarn="${_permwarn}${_perm}\n" + fi + done + + if [ -n "$_permwarn" ]; then + echo "Notice: If you are planning to use the provided systemd service units for" + echo "Notice: confluent-ksql, make sure that read-write permissions" + echo "Notice: for user $_user and group $_group are set up according to the" + echo "Notice: following commands:" + /bin/echo -e "$_permwarn" + fi + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/confluent-ksql.service b/debian/confluent-ksql.service new file mode 100644 index 000000000000..910b0582b17e --- /dev/null +++ b/debian/confluent-ksql.service @@ -0,0 +1,13 @@ +[Unit] +Description=Streaming SQL engine for Apache Kafka +Documentation=http://docs.confluent.io/ +After=network.target confluent-kafka.target confluent-schema-registry.target + +[Service] +Type=simple +User=cp-ksql +Group=confluent +Environment="LOG_DIR=/var/log/confluent/ksql" +ExecStart=/usr/bin/ksql-start /etc/ksql/ksql.properties +TimeoutStopSec=180 +Restart=no diff --git a/debian/confluent-ksql.spec.in b/debian/confluent-ksql.spec.in new file mode 100644 index 000000000000..b72cfdce0075 --- /dev/null +++ b/debian/confluent-ksql.spec.in @@ -0,0 +1,101 @@ +Summary: Streaming SQL engine for Kafka +Name: confluent-ksql +Version: ##RPMVERSION## +Release: ##RPMRELEASE## +Group: Applications/Internet +License: Apache (v2) +Source0: confluent-ksql-%{version}.tar.gz +URL: http://confluent.io +BuildRoot: %{_tmppath}/%{name}-%{version}-root +Vendor: Confluent, Inc. +Packager: Confluent Packaging +BuildArch: noarch + +Requires: confluent-rest-utils + +%description + +Streaming SQL engine for Apache Kafka. + +%define __jar_repack %{nil} + +%pre + +_user=cp-ksql +_group=confluent + +getent group $_group 2>&1 >/dev/null || groupadd -r $_group +getent passwd $_user 2>&1 >/dev/null || \ + useradd -r -g $_group --home-dir /tmp --no-create-home -s /sbin/nologin -c "Confluent KSQL" $_user + +_permwarn= +for dir in /var/log/confluent ; do + if [ $dir = /var/log/confluent ]; then + # Confluent log directory should be writable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+wx,o= $dir" + else + # Other dirs are only readable by group + _perm="chown ${_user}:${_group} $dir && chmod u+wx,g+r,o= $dir" + fi + + if [ ! -d $dir ]; then + echo "Creating directory $dir with owner $_user:$_group" + mkdir -p $dir + eval $_perm + else + echo "Notice: Not creating existing directory $dir, ensure proper permissions for user $_user group $_group" + _permwarn="${_permwarn}${_perm}\n" + fi +done + +if [ -n "$_permwarn" ]; then + echo "Notice: If you are planning to use the provided systemd service units for" + echo "Notice: confluent-ksql, make sure that read-write permissions" + echo "Notice: for user $_user and group $_group are set up according to the" + echo "Notice: following commands:" + /bin/echo -e "$_permwarn" +fi + +%post + +%preun + +%postun + +%prep + +%setup + +%build + +%install + +# Clean out any previous builds not on slash +[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} + +%{__mkdir_p} %{buildroot} +%{__cp} -R * %{buildroot} +# The spec file gets included, get rid of it +%{__rm} %{buildroot}/confluent-ksql.spec + +%{__mkdir_p} %{buildroot}/var/log/confluent + + +%files +%defattr(-,root,root) +/usr/bin/* +/usr/share/java/ksql +/lib/systemd/system/ + +%doc +/usr/share/doc/ksql + +%config(noreplace) /etc/ksql/* + +%clean +#used to cleanup things outside the build area and possibly inside. +[ "%{buildroot}" != "/" ] && %{__rm} -rf %{buildroot} + +%changelog +* Fri Jan 26 2018 Confluent Packaging +- Initial import From 1ac9ba2c36dd799de801cf5bf3b1f366cbd61300 Mon Sep 17 00:00:00 2001 From: Hojjat Jafarpour Date: Fri, 26 Jan 2018 12:57:29 -0800 Subject: [PATCH 5/7] Updated log directories. --- config/log4j-file.properties | 2 +- config/log4j-rolling.properties | 6 +++--- debian/confluent-ksql.service | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/log4j-file.properties b/config/log4j-file.properties index 7158bbd55664..1fbd8f468394 100644 --- a/config/log4j-file.properties +++ b/config/log4j-file.properties @@ -6,6 +6,6 @@ log4j.rootLogger=WARN, default.file log4j.appender.default.file=io.confluent.ksql.util.TimestampLogFileAppender log4j.appender.default.file.ImmediateFlush=true log4j.appender.default.file.append=false -log4j.appender.default.file.file=/tmp/ksql-logs/cli-%timestamp.log +log4j.appender.default.file.file=${ksql.log.dir}/ksql-logs/cli-%timestamp.log log4j.appender.default.file.layout=org.apache.log4j.PatternLayout log4j.appender.default.file.layout.ConversionPattern=[%d] %p %m (%c:%L)%n \ No newline at end of file diff --git a/config/log4j-rolling.properties b/config/log4j-rolling.properties index 900c42ad7b6e..681864f324ca 100644 --- a/config/log4j-rolling.properties +++ b/config/log4j-rolling.properties @@ -5,7 +5,7 @@ log4j.rootLogger=INFO, main # appenders log4j.appender.main=org.apache.log4j.RollingFileAppender -log4j.appender.main.File=/tmp/ksql.log +log4j.appender.main.File=${ksql.log.dir}/ksql.log log4j.appender.main.layout=org.apache.log4j.PatternLayout log4j.appender.main.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.appender.main.MaxFileSize=10MB @@ -13,12 +13,12 @@ log4j.appender.main.MaxBackupIndex=5 log4j.appender.main.append=true log4j.appender.streams=org.apache.log4j.RollingFileAppender -log4j.appender.streams.File=/tmp/ksql-streams.log +log4j.appender.streams.File=${ksql.log.dir}/ksql-streams.log log4j.appender.streams.layout=org.apache.log4j.PatternLayout log4j.appender.streams.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.appender.kafka=org.apache.log4j.RollingFileAppender -log4j.appender.kafka.File=/tmp/ksql-kafka.log +log4j.appender.kafka.File=${ksql.log.dir}/ksql-kafka.log log4j.appender.kafka.layout=org.apache.log4j.PatternLayout log4j.appender.kafka.layout.ConversionPattern=[%d] %p %m (%c:%L)%n log4j.appender.kafka.MaxFileSize=10MB diff --git a/debian/confluent-ksql.service b/debian/confluent-ksql.service index 910b0582b17e..ca2694d05e16 100644 --- a/debian/confluent-ksql.service +++ b/debian/confluent-ksql.service @@ -8,6 +8,6 @@ Type=simple User=cp-ksql Group=confluent Environment="LOG_DIR=/var/log/confluent/ksql" -ExecStart=/usr/bin/ksql-start /etc/ksql/ksql.properties +ExecStart=/usr/bin/ksql-server-start /etc/ksql/ksqlserver.properties TimeoutStopSec=180 Restart=no From 1614692061fb7bb1746f1047d967d68e12f8d20e Mon Sep 17 00:00:00 2001 From: Hojjat Jafarpour Date: Fri, 26 Jan 2018 13:33:53 -0800 Subject: [PATCH 6/7] honor LOG_DIR. --- bin/ksql-run-class | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/bin/ksql-run-class b/bin/ksql-run-class index e94b4dc54810..7b17d8000583 100755 --- a/bin/ksql-run-class +++ b/bin/ksql-run-class @@ -8,17 +8,11 @@ set -ue #cd -P deals with symlink from /bin to /usr/bin base_dir=$( cd -P "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd ) -# Log directory to use -if [ -z "${LOG_DIR:-}" ]; then - LOG_DIR="/tmp/ksql-logs" +if [[ -n $LOG_DIR ]]; then + [[ -d $LOG_DIR ]] || mkdir -p "$LOG_DIR" + KSQL_LOG4J_OPTS="-Dksql.log.dir=$LOG_DIR ${KSQL_LOG4J_OPTS}" fi -# create logs directory -if [ ! -d "$LOG_DIR" ]; then - mkdir -p "$LOG_DIR" -fi - - : "${KSQL_CLASSPATH:=""}" : "${KSQL_LOG4J_OPTS:=""}" : "${KSQL_JMX_OPTS:=""}" From 37e902b965773d6dcc76fc55c9b8068ada5acddf Mon Sep 17 00:00:00 2001 From: Hojjat Jafarpour Date: Fri, 26 Jan 2018 13:43:46 -0800 Subject: [PATCH 7/7] Set the default for LOG_DIR if it's not set. --- bin/ksql-run-class | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/ksql-run-class b/bin/ksql-run-class index 7b17d8000583..c6bc0c9f5286 100755 --- a/bin/ksql-run-class +++ b/bin/ksql-run-class @@ -8,6 +8,12 @@ set -ue #cd -P deals with symlink from /bin to /usr/bin base_dir=$( cd -P "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd ) +# Log directory to use +if [ -z "${LOG_DIR:-}" ]; then + LOG_DIR="/tmp/ksql-logs" +fi + +# create logs directory if [[ -n $LOG_DIR ]]; then [[ -d $LOG_DIR ]] || mkdir -p "$LOG_DIR" KSQL_LOG4J_OPTS="-Dksql.log.dir=$LOG_DIR ${KSQL_LOG4J_OPTS}"