diff --git a/cron.hourly b/cron.hourly new file mode 100644 index 0000000000000000000000000000000000000000..0ef168efff583670846c7e96a1373487f86d13e6 --- /dev/null +++ b/cron.hourly @@ -0,0 +1,3 @@ +#! /bin/sh + +runuser -c '/usr/libexec/resalloc-merge-hook-logs' - resalloc diff --git a/files_in_srpm.txt b/files_in_srpm.txt new file mode 100644 index 0000000000000000000000000000000000000000..1010237db6c3c91632f4c095328d89e682d87ada --- /dev/null +++ b/files_in_srpm.txt @@ -0,0 +1,8 @@ +cron.hourly +logrotate +merge-hook-logs +resalloc-5.9.tar.gz +resalloc-agent-spawner.service +resalloc.service +resalloc.spec +wait-for-ssh diff --git a/logrotate b/logrotate new file mode 100644 index 0000000000000000000000000000000000000000..f1ff9a397c3775492adbb87bd55a5e7365a9e4f6 --- /dev/null +++ b/logrotate @@ -0,0 +1,15 @@ +compress + +/var/log/resallocserver/main.log { + missingok + notifempty + rotate 4 + weekly + copytruncate +} + +/var/log/resallocserver/hooks.log { + missingok + rotate 7 + daily +} diff --git a/merge-hook-logs b/merge-hook-logs new file mode 100755 index 0000000000000000000000000000000000000000..348527a98e4c929ccdf46aaf3a73efcde32467c5 --- /dev/null +++ b/merge-hook-logs @@ -0,0 +1,17 @@ +#! /bin/sh + +export PATH=/usr/bin + +hooks_log=/var/log/resallocserver/hooks.log + +find /var/log/resallocserver/hooks -type f -mmin +60 \ + | sort | \ +while read -r file; do + { + echo "=== $file ===" + cat "$file" + echo + echo + } >> "$hooks_log" + rm "$file" +done diff --git a/resalloc-5.9.tar.gz b/resalloc-5.9.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..24e29d399acf5a6aac317273f173b9d96ef55c12 Binary files /dev/null and b/resalloc-5.9.tar.gz differ diff --git a/resalloc-agent-spawner.service b/resalloc-agent-spawner.service new file mode 100644 index 0000000000000000000000000000000000000000..b4bb2c6d3a64cd984be5b7e024561dbffd6efd44 --- /dev/null +++ b/resalloc-agent-spawner.service @@ -0,0 +1,15 @@ +[Unit] +Description=Start self-stending agent-like resources using Resalloc +After=syslog.target network.target auditd.service + +[Service] +Type=simple +User=resalloc +Group=resalloc +ExecStart=/usr/bin/resalloc-agent-spawner +# we don't want to kill background action processors (daemoncontext) +KillMode=process +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/resalloc.service b/resalloc.service new file mode 100644 index 0000000000000000000000000000000000000000..e29ae38c5c4b83dd45defed9d71414a276e002f8 --- /dev/null +++ b/resalloc.service @@ -0,0 +1,16 @@ +[Unit] +Description=Resource allocator server +After=network.target + +[Service] +Type=simple +Restart=always +RestartSec=5 +User=resalloc +Group=resalloc + +Environment=CONFIG_DIR=/etc/resallocserver +ExecStart=/usr/bin/resalloc-server + +[Install] +WantedBy=multi-user.target diff --git a/resalloc.spec b/resalloc.spec new file mode 100644 index 0000000000000000000000000000000000000000..69f5ef7183af2712be241642cd2afd3156f9a4d0 --- /dev/null +++ b/resalloc.spec @@ -0,0 +1,277 @@ +%global srcname resalloc + +%global sysuser resalloc +%global sysgroup %sysuser +%global _logdir %_var/log/%{name}server +%global _homedir %_sharedstatedir/%{name}server + +%global agent_user resalloc-agent-spawner +%global agent_group %agent_user + +%global create_user_group() \ +getent group "%1" >/dev/null || groupadd -r "%1" \ +getent passwd "%1" >/dev/null || \\\ +useradd -r -g "%2" -G "%2" -s "%3" \\\ + -c "%1 service user" "%1" \\\ + -d "%4" + + +%global sum Resource allocator for expensive resources +%global desc \ +The resalloc project aims to help with taking care of dynamically \ +allocated resources, for example ephemeral virtual machines used for \ +the purposes of CI/CD tasks. + +Name: %srcname +Summary: %sum - client tooling +Version: 5.9 +Release: 1 +License: GPL-2.0-or-later +URL: https://github.com/praiskup/resalloc +BuildArch: noarch + +BuildRequires: make +BuildRequires: postgresql-server + + +BuildRequires: python3-alembic +BuildRequires: python3-argparse-manpage +BuildRequires: python3-devel +BuildRequires: python3-psycopg2 +BuildRequires: python3-pytest +BuildRequires: python3-pytest-cov +BuildRequires: python3-setuptools +BuildRequires: python3-six +BuildRequires: python3-sqlalchemy +BuildRequires: python3-pyyaml +BuildRequires: systemd + +Requires: python3-%srcname = %version-%release + +Source0: https://github.com/praiskup/%name/releases/download/v%version/%name-%version.tar.gz +Source1: resalloc.service +Source2: logrotate +Source3: merge-hook-logs +Source4: cron.hourly +Source5: resalloc-agent-spawner.service +# GPL-2.0-or-later too +Source6: https://raw.githubusercontent.com/praiskup/wait-for-ssh/main/wait-for-ssh + +%description +%desc + +The %name package provides the client-side tooling. + + +%package server +Summary: %sum - server part + +Requires: crontabs +Requires: logrotate +Requires: python3-%srcname = %version-%release +Requires: %srcname-helpers = %version-%release +Requires: python3-alembic +Requires: python3-six +Requires: python3-sqlalchemy +Requires: python3-pyyaml + +Requires(pre): /usr/sbin/useradd +%description server +%desc + +The %name-server package provides the resalloc server, and +some tooling for resalloc administrators. + + +%package helpers +Summary: %sum - helper/library scripts + +%description helpers +%desc + +Helper and library-like scripts for external Resalloc plugins like resalloc-aws, +resalloc-openstack, etc. + +%package webui +Summary: %sum - webui part + +Requires: python3-%srcname = %version-%release +Requires: %name-server +Requires: python3-flask +Recommends: %name-selinux + +%description webui +%desc + +The %name-webui package provides the resalloc webui, +it shows page with information about resalloc resources. + +%package agent-spawner +Summary: %sum - daemon starting agent-like resources + +Requires(pre): /usr/sbin/useradd +Requires: python3-copr-common >= 0.23 +Requires: python3-daemon +Requires: python3-redis +Requires: python3-resalloc +Requires: python3-setproctitle + +%description agent-spawner +%desc + +Agent Spawner maintains sets resources (agents) of certain kind and in certain +number, according to given configuration. Typical Resalloc resource is +completely dummy, fully controlled from the outside. With agent-like resources +this is different — such resources are self-standing, they take care of +themselves, perhaps interacting/competing with each other. The only thing that +agent-spawner needs to do is to control the ideal number of them. + + +%package -n python3-%srcname +Summary: %sum - Python 3 client library +%{?python_provide:%python_provide python3-%srcname} +%description -n python3-%srcname +%desc + +The python3-%name package provides Python 3 client library for talking +to the resalloc server. + + +%package selinux +Summary: SELinux module for %{name} +Requires: %name-webui = %version-%release +# Requires(post): policycoreutils-python +BuildRequires: selinux-policy-devel +%{?selinux_requires} + +%description selinux +%desc + +%post selinux +semanage fcontext -a -t httpd_sys_script_exec_t \ + %_var/www/cgi-%{name} 2>/dev/null || : +restorecon -R %_var/www/cgi-%{name} || : + + +%prep +%autosetup -p1 -n %name-%version + + +%build +%py3_build +sed "1c#! $python" %SOURCE6 > %{name}-wait-for-ssh + + +%install +%py3_install +install -d -m 755 %buildroot%_datadir/%{name}webui +cp -r %{name}webui/templates %buildroot%_datadir/%{name}webui/ +cp -r %{name}webui/static %buildroot%_datadir/%{name}webui/ + +install -d -m 755 %buildroot%_var/www/ +install -p -m 755 %{name}webui/cgi-resalloc %buildroot%_var/www/cgi-%{name} + +mkdir -p %buildroot%_unitdir +mkdir -p %buildroot%_logdir +install -p -m 644 %SOURCE1 %buildroot%_unitdir +install -p -m 644 %SOURCE5 %buildroot%_unitdir +install -d -m 700 %buildroot%_homedir +install -d -m 700 %buildroot%_sysconfdir/logrotate.d +install -p -m 644 %SOURCE2 %buildroot%_sysconfdir/logrotate.d/resalloc-server +install -p -m 644 man/resalloc-server.1 %buildroot%_mandir/man1 +install -d -m 755 %buildroot/%_libexecdir +install -p -m 755 %SOURCE3 %buildroot/%_libexecdir/%name-merge-hook-logs +install -d %buildroot%_sysconfdir/cron.hourly +install -p -m 755 %SOURCE4 %buildroot%_sysconfdir/cron.hourly/resalloc +install -p -m 755 %name-wait-for-ssh %buildroot%_bindir/%name-wait-for-ssh + +# for now EUR call mock by root user, just skip check +# need re-enable check when builder run as non-root user +#%check +make check TEST_PYTHONS="python3" + + +# Simplify "alembic upgrade head" actions. +ln -s "%{python3_sitelib}/%{name}server" %buildroot%_homedir/project + + +%pre server +%create_user_group %sysuser %sysgroup /bin/bash %_homedir + +%post server +%systemd_post resalloc.service + +%postun server +%systemd_postun_with_restart resalloc.service + +%pre agent-spawner +%create_user_group %agent_user %agent_group /bin/false / + +%post agent-spawner +%systemd_post resalloc-agent-spawner.service + +%postun agent-spawner +%systemd_postun_with_restart resalloc-agent-spawner.service + +%global doc_files NEWS README.md + +%files +%doc %doc_files +%license COPYING +%{_bindir}/%{name} +%_mandir/man1/%{name}.1* + + +%files -n python3-%srcname +%doc %doc_files +%license COPYING +%{python3_sitelib}/%{name} +%{python3_sitelib}/%{name}-*.egg-info + + +%files server +%doc %doc_files +%license COPYING +%{python3_sitelib}/%{name}server +%{_bindir}/%{name}-server +%{_bindir}/%{name}-maint +%attr(0750, %sysuser, %sysgroup) %dir %{_sysconfdir}/%{name}server +%config(noreplace) %{_sysconfdir}/%{name}server/* +%_unitdir/resalloc.service +%attr(0700, %sysuser, %sysgroup) %dir %_logdir +%_mandir/man1/%{name}-maint.1* +%_mandir/man1/%{name}-server.1* +%attr(0700, %sysuser, %sysgroup) %_homedir +%config %_sysconfdir/logrotate.d/resalloc-server +%_libexecdir/resalloc-merge-hook-logs +%config %attr(0755, root, root) %{_sysconfdir}/cron.hourly/resalloc + + +%files helpers +%doc %doc_files +%license COPYING +%{_bindir}/%{name}-check-vm-ip +%{_bindir}/%{name}-wait-for-ssh + +%files agent-spawner +%_bindir/resalloc-agent* +%{python3_sitelib}/%{name}_agent_spawner +%_unitdir/resalloc-agent-spawner.service +%config(noreplace) %_sysconfdir/resalloc-agent-spawner + +%files webui +%doc %doc_files +%license COPYING +%{python3_sitelib}/%{name}webui/ +%_datadir/%{name}webui/ +%_var/www/cgi-%{name} + +%files selinux + + +%changelog +* Wed Mar 12 2025 lichaoran - 5.9-1 +- upgrade to 5.9 + +* Tue Mar 21 2023 lichaoran - 4.91 +- init package diff --git a/resalloc.src.rpm b/resalloc.src.rpm new file mode 100644 index 0000000000000000000000000000000000000000..bb222b460004ccbb420d548aacfe82f3e8ab1f4e Binary files /dev/null and b/resalloc.src.rpm differ diff --git a/wait-for-ssh b/wait-for-ssh new file mode 100644 index 0000000000000000000000000000000000000000..d302c775f7f685c13b0c9503788c7b00c7916f84 --- /dev/null +++ b/wait-for-ssh @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (C) 2017 Pavel Raiskup +# +# This program accepts one argument IP or HOSTNAME. First try to connect to the +# HOSTNAME as 'root' user. If cloud-init scripts instruct us to use different +# user than 'root', switch to that user and check again. In the end, print the +# successful username on stdout. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from re import compile as re_compile +import sys +from os import devnull +from threading import Thread, Event +from argparse import ArgumentParser +from subprocess import Popen, PIPE +import logging + +handler = logging.StreamHandler() +log = logging.getLogger() +log.setLevel(logging.INFO) +log.addHandler(handler) + +# create console handler and set level to debug + +ssh = [ + 'ssh', + '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'PasswordAuthentication=no', + '-o', 'ConnectTimeout=10', +] + +expected_output = 'foobar' +inner_cmd = 'echo ' + expected_output + + +class Checker(Thread): + user = 'root' + daemon = True + user_re = '[a-zA-Z0-9_.][a-zA-Z0-9_.-]*[$]?' + re_clouduser = re_compile('Please login as the user "({0})"'.format(user_re)) + event = Event() + + def loop(self): + cmd = ssh + [ + '{0}@{1}'.format(self.user, self.args.host), + inner_cmd, + ] + + with open(devnull, 'w') as drop: + log.debug('executing: ' + ' '.join(cmd)) + self.child = Popen(cmd, stdout=PIPE, stderr=drop) + (stdout, _) = self.child.communicate() + + exp = (expected_output + '\n').encode('ascii') + if self.child.returncode == 0 and stdout == exp: + if self.args.print_user: + print(self.user) + return True + + if self.args.cloud_user: + match = self.re_clouduser.search(str(stdout)) + if match: + self.user = match.group(1) + log.info('cloud user switched to ' + self.user) + return False + + def run(self): + while True: + if self.loop(): + # Success! + break + + if self.event.wait(1): + log.debug("stopping per kill event") + break + + def kill(self): + self.event.set() + + # Best effort kill. + try: + self.child.kill() + except: + pass + self.join() + + +parser = ArgumentParser( + description="Wait till the host's ssh becomes responsive.") +parser.add_argument('host', help='hostname or IP') +parser.add_argument('--timeout', + help='seconds to wait before failure, default=indefinitely', + default=None, type=float) +parser.add_argument('--check-cloud-user', action='store_true', default=False, + dest='cloud_user', + help='if cloud-init disallows "root" login, try to detect the cloud ' \ + +'user and use that') +parser.add_argument('--print-user', action='store_true', default=False, + dest='print_user', + help='print the username which succeeded to connect on stdout') +parser.add_argument('--log', default=False, + dest='log_verbosity', + help='set the threshold for logging, e.g. debug, info, error, ...') + + +def main(): + sleep_period = 1.0 + args = parser.parse_args() + + if args.log_verbosity: + log.setLevel(logging.getLevelName(args.log_verbosity.upper())) + + def timeouted(): + if args.timeout is None: + return False + log.debug("wait {0}s, remains {1}s".format(sleep_period, args.timeout)) + args.timeout -= sleep_period + return args.timeout <= 0 + + checker = Checker() + checker.args = args + checker.start() + + try: + # threading.join() is not Ctrl-C interruptable :( in python2, so we need + # this ugly infinite loop. + # https://stackoverflow.com/questions/25676835/signal-handling-in-multi-threaded-python + while True: + checker.join(sleep_period) + if not checker.is_alive(): + # Success! + return 0 + + if timeouted(): + log.error("timeout!") + checker.kill() + return 1 + + except KeyboardInterrupt: + log.error("interrupt by user") + checker.kill() + return 1 + +if __name__ == "__main__": + sys.exit(main())