From e1ab3d0a2bc351f80d38bb53f23dd7b61bc518b3 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 29 Dec 2018 14:11:43 +0100 Subject: [PATCH 01/17] Make changes for system test ansible setup --- icpc-wf/ansible/common_tasks_build.yml | 4 +- icpc-wf/ansible/common_tasks_prebuild.yml | 73 ------------------- icpc-wf/ansible/domserver.yml | 62 +++++++--------- .../ansible/files/judgedaemon-systest.service | 16 ++++ icpc-wf/ansible/files/judgedaemon.service | 20 ----- icpc-wf/ansible/hosts | 14 +--- icpc-wf/ansible/judgehost.yml | 34 +-------- icpc-wf/ansible/variables.yml | 18 ++--- 8 files changed, 56 insertions(+), 185 deletions(-) create mode 100644 icpc-wf/ansible/files/judgedaemon-systest.service delete mode 100644 icpc-wf/ansible/files/judgedaemon.service diff --git a/icpc-wf/ansible/common_tasks_build.yml b/icpc-wf/ansible/common_tasks_build.yml index d39fc853..7df18be7 100644 --- a/icpc-wf/ansible/common_tasks_build.yml +++ b/icpc-wf/ansible/common_tasks_build.yml @@ -3,7 +3,7 @@ - name: run maintainer-conf become: yes become_user: domjudge - command: make maintainer-conf CONFIGURE_FLAGS='--disable-doc-build' + command: make maintainer-conf CONFIGURE_FLAGS='--disable-doc-build --with-judgehost_chrootdir=/chroot/domjudge-systest' register: dj_configured args: chdir: "{{DJ_DIR}}" @@ -23,5 +23,5 @@ shell: make -C {{DJ_DIR}} maintainer-postinstall-permissions - name: copy domjudge-sudoers file - copy: remote_src=True src={{DJ_DIR}}/etc/sudoers-domjudge dest=/etc/sudoers.d/domjudge mode=0440 owner=root group=root + copy: remote_src=True src={{DJ_DIR}}/etc/sudoers-domjudge dest=/etc/sudoers.d/domjudge-systest mode=0440 owner=root group=root diff --git a/icpc-wf/ansible/common_tasks_prebuild.yml b/icpc-wf/ansible/common_tasks_prebuild.yml index ed2fa40a..7342894d 100644 --- a/icpc-wf/ansible/common_tasks_prebuild.yml +++ b/icpc-wf/ansible/common_tasks_prebuild.yml @@ -1,17 +1,5 @@ # Common tasks before building DOMjudge. --- - - name: add domjudge to hosts file - lineinfile: - dest: /etc/hosts - regexp: 'domserver$' - line: "{{DOMSERVER_IP}} domserver" - - - name: set timezone - timezone: - name: "{{TIMEZONE}}" - - - include: common_tasks_packages_icpc-wf.yml - - name: install common required/useful packages tags: packages apt: pkg={{item}} state=present @@ -83,13 +71,6 @@ - { name: 'email', value: 'team@domjudge.org' } - { name: 'name', value: 'DOMjudge team' } - - name: Allow 'sudo' group to have passwordless sudo - lineinfile: - dest: /etc/sudoers - state: present - regexp: '^%sudo' - line: '%sudo ALL=(ALL) NOPASSWD: ALL' - - name: Create .ssh directory file: path="/home/domjudge/.ssh" group=domjudge owner=domjudge mode=0700 state=directory @@ -151,57 +132,3 @@ - name: configure domjudge logrotate copy: src=files/logrotate.domjudge dest=/etc/logrotate.d/domjudge - - - name: copy DOMjudge logo binary - copy: src=files/domlogo dest=/home/domjudge/domlogo owner=domjudge group=domjudge mode=0755 - - - name: make sure lightdm config directory exists - file: path=/etc/lightdm/lightdm.conf.d state=directory - - - name: enable GDM autologin - lineinfile: - path: /etc/gdm3/custom.conf - regexp: 'AutomaticLoginEnable' - line: 'AutomaticLoginEnable=true' - notify: restart gdm - - - name: Automatically login domjudge user - lineinfile: - path: /etc/gdm3/custom.conf - regexp: 'AutomaticLogin' - line: 'AutomaticLogin=domjudge' - notify: restart gdm - - - name: make sure autostart directory exists - file: dest=/home/domjudge/.config/autostart state=directory owner=domjudge group=domjudge - tags: fix_autostart - - - name: install SSL server certificates - copy: - src: "{{ item }}" - dest: /etc/ssl/certs/ - owner: root - group: root - mode: 0644 - with_fileglob: - - files/ssl/*.crt - notify: update-ca-certificates - - - name: create ca certificates shared directory - file: - dest: /usr/local/share/ca-certificates - state: directory - owner: root - group: root - - - name: install SSL server certificates into CA certificates - copy: - src: "{{ item }}" - dest: /usr/local/share/ca-certificates - owner: root - group: root - mode: 0644 - with_fileglob: - - files/ssl/*.crt - notify: update-ca-certificates - diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index e24cf84f..4c5e4ac3 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -20,17 +20,12 @@ service: name=rsyslog enabled=yes state=restarted - name: restart systemctl shell: systemctl daemon-reload - - name: restart gdm - service: name=gdm3 enabled=yes state=restarted tasks: - name: include global variables include_vars: variables.yml - include: common_tasks_prebuild.yml - - name: template out the restapi secret file - template: src=files/restapi.secret-domserver.j2 dest={{DJ_DIR}}/etc/restapi.secret owner=domjudge group=domjudge mode=0600 - - name: install domserver required packages apt: pkg={{item}} state=present with_items: @@ -124,28 +119,17 @@ dest: "{{DJ_DIR}}/webapp/web/style.css" when: BACKGROUND_COLOR is defined - - name: install SSL private key files - copy: - src: "{{ item }}" - dest: /etc/ssl/private/ - owner: root - group: root - mode: 0600 - with_fileglob: - - files/ssl/*.key - notify: update-ca-certificates - - name: copy in domjudge FPM conf copy: src={{DJ_DIR}}/etc/domjudge-fpm.conf remote_src=yes dest=/etc/php/7.2/fpm/pool.d/domjudge.conf notify: restart PHP FPM - name: copy in domjudge nginx conf - copy: src={{DJ_DIR}}/etc/nginx-conf remote_src=yes dest=/etc/nginx/sites-available/domjudge.conf + copy: src={{DJ_DIR}}/etc/nginx-conf remote_src=yes dest=/etc/nginx/sites-available/domjudge-systest notify: restart nginx - name: remove HTTP host blockinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest marker: "{mark}" marker_begin: "### http host config ###" marker_end: "# Alternatively, use HTTPS and redirect HTTP to HTTPS:" @@ -153,43 +137,58 @@ - name: enable HTTP redirect and HTTPS blocks replace: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest after: 'upstream domjudge \{' regexp: '^# ?(.*)' replace: '\1' notify: restart nginx + - name: remove HTTP redirect host + blockinfile: + path: /etc/nginx/sites-available/domjudge-systest + marker: "{mark}" + marker_begin: listen 80;" + marker_end: "server {" + notify: restart nginx + + - name: Set system test hostname + lineinfile: + path: /etc/nginx/sites-available/domjudge-systest + regexp: 'server_name' + line: "\tserver_name systest.domjudge.org;" + notify: restart nginx + - name: remove IPv6 listens lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'listen\s+\[.*\]:\d+;' state: absent notify: restart nginx - name: change IPv4 HTTP listen to default server lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'listen.*80;' line: "\tlisten 80 default_server;" notify: restart nginx - name: change IPv4 HTTPS listen to all interfaces lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'listen.*443;' line: "\tlisten 443 ssl http2 default_server;" notify: restart nginx - name: increase max body size to upload size lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'client_max_body_size' line: "\tclient_max_body_size {{PHP_UPLOAD_MAX_FILESIZE}};" notify: restart nginx - name: remove prefixed domjudge location blockinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest marker: "{mark}" marker_begin: " # Or you can install it with a prefix" marker_end: " }" @@ -197,20 +196,20 @@ - name: configure SSL certificate lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'ssl_certificate (.*)' line: "\tssl_certificate {{DOMSERVER_SSL_CERT}};" notify: restart nginx - name: configure SSL key lineinfile: - path: /etc/nginx/sites-available/domjudge.conf + path: /etc/nginx/sites-available/domjudge-systest regexp: 'ssl_certificate_key (.*)' line: "\tssl_certificate_key {{DOMSERVER_SSL_KEY}};" notify: restart nginx - name: enable nginx conf for domjudge - file: src=/etc/nginx/sites-available/domjudge.conf dest=/etc/nginx/sites-enabled/domjudge.conf state=link + file: src=/etc/nginx/sites-available/domjudge-systest dest=/etc/nginx/sites-enabled/domjudge-systest state=link notify: restart nginx - name: disable default nginx site @@ -228,12 +227,3 @@ - { key: 'php_admin_value[upload_max_filesize]', regexp: '^php_admin_value\[upload_max_filesize\]', value: '{{PHP_UPLOAD_MAX_FILESIZE}}' } - { key: 'php_admin_value[post_max_size]', regexp: '^php_admin_value\[post_max_size\]', value: '{{PHP_POST_MAX_SIZE}}' } notify: restart PHP FPM - - - name: add autostart shortcuts - copy: src=files/{{item}}.desktop dest=/home/domjudge/.config/autostart/ owner=domjudge group=domjudge mode=0755 - with_items: - - htop - - taillog-domserver-nginx-error - - taillog-domserver-symfony-error - - domjudgelogo-domserver - diff --git a/icpc-wf/ansible/files/judgedaemon-systest.service b/icpc-wf/ansible/files/judgedaemon-systest.service new file mode 100644 index 00000000..14c851b8 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon-systest.service @@ -0,0 +1,16 @@ +[Unit] +Description=DOMjudge JudgeDaemon for System Test +After=network.target + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 0 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon.service b/icpc-wf/ansible/files/judgedaemon.service deleted file mode 100644 index ac3176bb..00000000 --- a/icpc-wf/ansible/files/judgedaemon.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=DOMjudge JudgeDaemon -Requires=create_cgroups.service -Requires=tune_cpu.service -After=create_cgroups.service -After=tune_cpu.service -After=network.target - -[Service] -Type=simple - -ExecStart=/home/domjudge/domjudge/bin/judgedaemon -n 0 -User=domjudge - -Restart=always -RestartSec=3 - -[Install] -WantedBy=multi-user.target - diff --git a/icpc-wf/ansible/hosts b/icpc-wf/ansible/hosts index b41f495a..27b0ff99 100644 --- a/icpc-wf/ansible/hosts +++ b/icpc-wf/ansible/hosts @@ -1,15 +1,5 @@ [domserver] -domjudge-primary ansible_host=10.3.3.215 ansible_user=root -domjudge-backup ansible_host=10.3.3.216 ansible_user=root +domjudge-primary ansible_host=calca ansible_user=root [judgehost] -domjudge-judgehost1 ansible_host=10.2.2.192 ansible_user=root -domjudge-judgehost2 ansible_host=10.2.2.193 ansible_user=root -domjudge-judgehost3 ansible_host=10.2.2.194 ansible_user=root -domjudge-judgehost4 ansible_host=10.2.2.195 ansible_user=root -domjudge-judgehost5 ansible_host=10.2.2.196 ansible_user=root -domjudge-judgehost6 ansible_host=10.2.2.197 ansible_user=root -domjudge-judgehost7 ansible_host=10.2.2.198 ansible_user=root -domjudge-judgehost8 ansible_host=10.2.2.199 ansible_user=root -domjudge-judgehost9 ansible_host=10.2.2.200 ansible_user=root -domjudge-judgehost10 ansible_host=10.2.2.201 ansible_user=root +domjudge-judgehost ansible_host=calca ansible_user=root diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index bacf57fb..f192accb 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -16,8 +16,6 @@ service: name=rsyslog enabled=yes state=restarted - name: restart systemctl shell: systemctl daemon-reload - - name: restart gdm - service: name=gdm3 enabled=yes state=restarted tasks: - name: include global variables include_vars: variables.yml @@ -53,9 +51,6 @@ line: '\1' backrefs: yes - - name: enable internal monitor - file: path=/usr/share/X11/xorg.conf.d/22-icpc.conf state=absent - - name: copy chroot DEB packages to install copy: src=files/install-chroot dest=/tmp/dj_ansible/ @@ -64,7 +59,7 @@ environment: DEBMIRROR: http://packages/ubuntu args: - creates: "/chroot/domjudge" + creates: "/chroot/domjudge-systest" - name: fix kernel parameters lineinfile: @@ -73,16 +68,11 @@ line: 'GRUB_CMDLINE_LINUX_DEFAULT="quiet splash cgroup_enable=memory swapaccount=1"' notify: update-grub - - name: copy script to disable turboboost and hyperthreading - copy: src=files/disable-turboboost_ht dest=/usr/local/sbin/ mode=0755 - - name: copy systemd unit files copy: src=files/{{item}}.service dest=/etc/systemd/system/ tags: updateservice with_items: - - create_cgroups - - tune_cpu - - judgedaemon + - judgedaemon-systest notify: restart systemctl - name: make sure systemctl is restarted @@ -91,22 +81,4 @@ - name: enable and restart the services we just copied service: name={{item}} enabled=yes state=restarted with_items: - - create_cgroups - - tune_cpu - - judgedaemon - - - name: add autostart shortcuts - copy: src=files/{{item}}.desktop dest=/home/domjudge/.config/autostart/ owner=domjudge group=domjudge mode=0755 - with_items: - - taillog - - rotate - - domjudgelogo - - - name: disable systemd timers - command: systemctl mask {{item}} - args: - creates: /etc/systemd/system/{{item}} - with_items: - - apt-daily-upgrade.timer - - apt-daily.timer - - systemd-tmpfiles-clean.timer + - judgedaemon-systest diff --git a/icpc-wf/ansible/variables.yml b/icpc-wf/ansible/variables.yml index 9d478490..98564a79 100644 --- a/icpc-wf/ansible/variables.yml +++ b/icpc-wf/ansible/variables.yml @@ -1,5 +1,5 @@ # Directory of the domjudge repository checkout. -DJ_DIR: /home/domjudge/domjudge +DJ_DIR: /opt/domjudge-systest # Branch to checkout and use. DJ_BRANCH: master @@ -9,13 +9,13 @@ DJ_BRANCH: master # URL and IP of domserver from judgehosts. A hostname 'domserver' with # DOMSERVER_IP will be added to the judgehost /etc/hosts file. -DOMSERVER: https://domjudge -DOMSERVER_IP: 10.3.3.215 +DOMSERVER: https://systest.domjudge.org +DOMSERVER_IP: 131.155.69.89 DOMSERVER_URL: "{{DOMSERVER}}" -DOMSERVER_SSL_CERT: /etc/ssl/certs/domserver.crt -DOMSERVER_SSL_KEY: /etc/ssl/private/domserver.key +DOMSERVER_SSL_CERT: /etc/letsencrypt/live/systest.domjudge.org/fullchain.pem +DOMSERVER_SSL_KEY: /etc/letsencrypt/live/systest.domjudge.org/privkey.pem -TIMEZONE: "Europe/Lisbon" +TIMEZONE: "Europe/Amsterdam" PHP_FPM_MAX_CHILDREN: 300 PHP_MEMORY_LIMIT: 1024M @@ -44,11 +44,7 @@ API_ADMIN_PASSWORD: API_ADMIN_PASSWORD #DJ_SHELL_USER_PW: some-hashed-password # Git repo URL -DJ_GIT_REPO: domjudge@10.3.3.223:domjudge - -# If using a Git repo which requires a SSH key, set the host and key here -DJ_GIT_HOST: 10.3.3.223 -DJ_GIT_SSH_KEY: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKayJQxbraoLvi7iYQ7TmTN08Spr1eFfcU/eqyfmacXDVSUMOn/LwwA0nh/bjkbtZSM6CFjQl2L9SXvlxSG9bYV2gYfOL4COxiVGPdVk783FcQlt3x+y4zFRZgz5FgJuGfRxWAqZstsjYceg1xQKkIFQLm+gup1EnLBcwKPARonDkRIa+5XYoKsGaRu1HFrzgGNIR1gmDXP1UAUgHz8MkELazNp1zTt7s7szFhNWhIdtBWbghrnRMss1W+qlx6umhd3T6y4EeJLxoUDYhbIQUCcBx+Rpf5sj/4LmgdCbHQS2OkXjaYtM4MGxEvfSrNT14rIV7HKrCr7BovVoj+p2El nicky@dyn070180.nbw.tue.nl +DJ_GIT_REPO: https://github.com/DOMjudge/domjudge # Key for domjudge user DJ_SSH_KEY: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEW4j7DOQ/BGT6ATtBLAcUHGVuHyydqs7E31DMbcX5uO icpc2018@domjudge-ccsadmin2 From 227b7fe4d8e1504725cb3ad341249b9c10e37b5f Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 29 Dec 2018 14:25:13 +0100 Subject: [PATCH 02/17] Calca uses MariaDB --- icpc-wf/ansible/domserver.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index 4c5e4ac3..fb5954ce 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -29,7 +29,7 @@ - name: install domserver required packages apt: pkg={{item}} state=present with_items: - - mysql-server + - mariadb-server - nginx - php-fpm - python-mysqldb From 9a7fd396a314df1433d20e42a9a8be8721c0046c Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 29 Dec 2018 15:12:11 +0100 Subject: [PATCH 03/17] Rename upstream for systest --- icpc-wf/ansible/domserver.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index fb5954ce..3b6472be 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -147,10 +147,24 @@ blockinfile: path: /etc/nginx/sites-available/domjudge-systest marker: "{mark}" - marker_begin: listen 80;" + marker_begin: " listen 80;" marker_end: "server {" notify: restart nginx + - name: rename upstream + lineinfile: + path: /etc/nginx/sites-available/domjudge-systest + regexp: '^upstream' + line: "upstream domjudgesystest {" + notify: restart nginx + + - name: rename fastcgi pass + lineinfile: + path: /etc/nginx/sites-available/domjudge-systest + regexp: 'fastcgi_pass' + line: "\t\tfastcgi_pass domjudgesystest;" + notify: restart nginx + - name: Set system test hostname lineinfile: path: /etc/nginx/sites-available/domjudge-systest From 9fac3fd77f972a9a4b0c4573d6084b9d4eeff832 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 29 Dec 2018 15:14:00 +0100 Subject: [PATCH 04/17] Remove unneeded nginx http default server --- icpc-wf/ansible/domserver.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index 3b6472be..4fd92da7 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -179,13 +179,6 @@ state: absent notify: restart nginx - - name: change IPv4 HTTP listen to default server - lineinfile: - path: /etc/nginx/sites-available/domjudge-systest - regexp: 'listen.*80;' - line: "\tlisten 80 default_server;" - notify: restart nginx - - name: change IPv4 HTTPS listen to all interfaces lineinfile: path: /etc/nginx/sites-available/domjudge-systest From 761c7f52c9e4687a0c8ee62e8d514d45c4d88de1 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 29 Dec 2018 15:15:56 +0100 Subject: [PATCH 05/17] Change order to fix fastcgi_pass --- icpc-wf/ansible/domserver.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index 4fd92da7..a6290467 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -158,13 +158,6 @@ line: "upstream domjudgesystest {" notify: restart nginx - - name: rename fastcgi pass - lineinfile: - path: /etc/nginx/sites-available/domjudge-systest - regexp: 'fastcgi_pass' - line: "\t\tfastcgi_pass domjudgesystest;" - notify: restart nginx - - name: Set system test hostname lineinfile: path: /etc/nginx/sites-available/domjudge-systest @@ -201,6 +194,13 @@ marker_end: " }" notify: restart nginx + - name: rename fastcgi pass + lineinfile: + path: /etc/nginx/sites-available/domjudge-systest + regexp: 'fastcgi_pass' + line: "\t\tfastcgi_pass domjudgesystest;" + notify: restart nginx + - name: configure SSL certificate lineinfile: path: /etc/nginx/sites-available/domjudge-systest From a20d3fb67116dd82307c0a67be80344bb7d66006 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 29 Dec 2018 15:20:19 +0100 Subject: [PATCH 06/17] =?UTF-8?q?Don=E2=80=99t=20use=20WF=20packages=20hos?= =?UTF-8?q?t=20for=20chroot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- icpc-wf/ansible/judgehost.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index f192accb..8a83ce5a 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -56,8 +56,6 @@ - name: create chroot shell: "{{DJ_DIR}}/misc-tools/dj_make_chroot -y -i openjdk-11-jdk-headless -l \"$(ls /tmp/dj_ansible/install-chroot/*.deb 2>/dev/null | tr '\n' ',')\" 2>&1 | tee /tmp/dj_make_chroot.log" - environment: - DEBMIRROR: http://packages/ubuntu args: creates: "/chroot/domjudge-systest" From 7c360499f0d5057425383f08bd8a1d4f334e094a Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 19 Jan 2019 10:11:18 +0100 Subject: [PATCH 07/17] Move server_name setting to correct file --- icpc-wf/ansible/domserver.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index 7dc6b1c7..60ff09af 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -164,9 +164,9 @@ - name: Set system test hostname lineinfile: - path: /etc/nginx/sites-available/domjudge-systest + path: /etc/nginx/snippets/domjudge-inner regexp: 'server_name' - line: "\tserver_name systest.domjudge.org;" + line: "server_name systest.domjudge.org;" notify: restart nginx - name: remove IPv6 listens From 18da9643f5a0257d5909c5832dc0a7e9603f07bb Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sat, 19 Jan 2019 10:12:30 +0100 Subject: [PATCH 08/17] Also move fastcgi_pass to correct file --- icpc-wf/ansible/domserver.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index 60ff09af..0de74517 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -216,9 +216,9 @@ - name: rename fastcgi pass lineinfile: - path: /etc/nginx/sites-available/domjudge-systest + path: /etc/nginx/snippets/domjudge-inner regexp: 'fastcgi_pass' - line: "\t\tfastcgi_pass domjudgesystest;" + line: "\tfastcgi_pass domjudgesystest;" notify: restart nginx - name: configure SSL certificate From f48ea4573015e0e3e71fdc1461c133e77b2ec508 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Sun, 27 Jan 2019 10:26:36 +0100 Subject: [PATCH 09/17] Also install php-intl as this is now required (cherry picked from commit e5078de01661ba7b18ddae27e2ea70e504e5bbfa) --- icpc-wf/ansible/common_tasks_prebuild.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/icpc-wf/ansible/common_tasks_prebuild.yml b/icpc-wf/ansible/common_tasks_prebuild.yml index 7342894d..a321ee67 100644 --- a/icpc-wf/ansible/common_tasks_prebuild.yml +++ b/icpc-wf/ansible/common_tasks_prebuild.yml @@ -23,6 +23,7 @@ - php-xml - php-zip - php-mbstring + - php-intl - bsdmainutils - libcgroup-dev - libcurl4-gnutls-dev From 4b2f27f77a4e516cdeec91843fb0bb55cd866156 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Mon, 28 Jan 2019 21:52:48 +0100 Subject: [PATCH 10/17] We will use a VM for the systest so update ansible to reflect that --- ...emon-systest.service => judgedaemon0.service} | 2 +- icpc-wf/ansible/files/judgedaemon1.service | 16 ++++++++++++++++ icpc-wf/ansible/files/judgedaemon2.service | 16 ++++++++++++++++ icpc-wf/ansible/files/judgedaemon3.service | 16 ++++++++++++++++ icpc-wf/ansible/files/judgedaemon4.service | 16 ++++++++++++++++ icpc-wf/ansible/files/judgedaemon5.service | 16 ++++++++++++++++ icpc-wf/ansible/hosts | 2 +- icpc-wf/ansible/judgehost.yml | 16 ++++++++++++++-- icpc-wf/ansible/variables.yml | 2 +- 9 files changed, 97 insertions(+), 5 deletions(-) rename icpc-wf/ansible/files/{judgedaemon-systest.service => judgedaemon0.service} (79%) create mode 100644 icpc-wf/ansible/files/judgedaemon1.service create mode 100644 icpc-wf/ansible/files/judgedaemon2.service create mode 100644 icpc-wf/ansible/files/judgedaemon3.service create mode 100644 icpc-wf/ansible/files/judgedaemon4.service create mode 100644 icpc-wf/ansible/files/judgedaemon5.service diff --git a/icpc-wf/ansible/files/judgedaemon-systest.service b/icpc-wf/ansible/files/judgedaemon0.service similarity index 79% rename from icpc-wf/ansible/files/judgedaemon-systest.service rename to icpc-wf/ansible/files/judgedaemon0.service index 14c851b8..d9ce12e5 100644 --- a/icpc-wf/ansible/files/judgedaemon-systest.service +++ b/icpc-wf/ansible/files/judgedaemon0.service @@ -1,5 +1,5 @@ [Unit] -Description=DOMjudge JudgeDaemon for System Test +Description=DOMjudge JudgeDaemon After=network.target [Service] diff --git a/icpc-wf/ansible/files/judgedaemon1.service b/icpc-wf/ansible/files/judgedaemon1.service new file mode 100644 index 00000000..ee074ea2 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon1.service @@ -0,0 +1,16 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 1 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon2.service b/icpc-wf/ansible/files/judgedaemon2.service new file mode 100644 index 00000000..552d4a7a --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon2.service @@ -0,0 +1,16 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 2 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon3.service b/icpc-wf/ansible/files/judgedaemon3.service new file mode 100644 index 00000000..85c4f201 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon3.service @@ -0,0 +1,16 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 3 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon4.service b/icpc-wf/ansible/files/judgedaemon4.service new file mode 100644 index 00000000..feb0d5d8 --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon4.service @@ -0,0 +1,16 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 4 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/files/judgedaemon5.service b/icpc-wf/ansible/files/judgedaemon5.service new file mode 100644 index 00000000..004573ce --- /dev/null +++ b/icpc-wf/ansible/files/judgedaemon5.service @@ -0,0 +1,16 @@ +[Unit] +Description=DOMjudge JudgeDaemon +After=network.target + +[Service] +Type=simple + +ExecStart=/opt/domjudge-systest/bin/judgedaemon -n 5 +User=domjudge + +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target + diff --git a/icpc-wf/ansible/hosts b/icpc-wf/ansible/hosts index 27b0ff99..a4fde50f 100644 --- a/icpc-wf/ansible/hosts +++ b/icpc-wf/ansible/hosts @@ -2,4 +2,4 @@ domjudge-primary ansible_host=calca ansible_user=root [judgehost] -domjudge-judgehost ansible_host=calca ansible_user=root +systest-judge ansible_host=systest-judge.domjudge.letstalk.nl ansible_user=root diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index 8a83ce5a..4abdda23 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -27,6 +27,8 @@ - domjudge-run-1 - domjudge-run-2 - domjudge-run-3 + - domjudge-run-4 + - domjudge-run-5 - name: create domjudge-run group group: name=domjudge-run state=present @@ -70,7 +72,12 @@ copy: src=files/{{item}}.service dest=/etc/systemd/system/ tags: updateservice with_items: - - judgedaemon-systest + - judgedaemon0 + - judgedaemon1 + - judgedaemon2 + - judgedaemon3 + - judgedaemon4 + - judgedaemon5 notify: restart systemctl - name: make sure systemctl is restarted @@ -79,4 +86,9 @@ - name: enable and restart the services we just copied service: name={{item}} enabled=yes state=restarted with_items: - - judgedaemon-systest + - judgedaemon0 + - judgedaemon1 + - judgedaemon2 + - judgedaemon3 + - judgedaemon4 + - judgedaemon5 diff --git a/icpc-wf/ansible/variables.yml b/icpc-wf/ansible/variables.yml index 03904c68..011eb915 100644 --- a/icpc-wf/ansible/variables.yml +++ b/icpc-wf/ansible/variables.yml @@ -2,7 +2,7 @@ DJ_DIR: /opt/domjudge-systest # Branch to checkout and use. -DJ_BRANCH: master +DJ_BRANCH: wf2019 # Set this to change the web interface background color. #BACKGROUND_COLOR: '#ddddff' From a9a9f91c8bfc24ca5193188abd2398e7cd52996e Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Tue, 29 Jan 2019 11:59:36 +0100 Subject: [PATCH 11/17] Make sure we use the pc2 repo --- icpc-wf/ansible/judgehost.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index 4abdda23..887cca73 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -58,6 +58,8 @@ - name: create chroot shell: "{{DJ_DIR}}/misc-tools/dj_make_chroot -y -i openjdk-11-jdk-headless -l \"$(ls /tmp/dj_ansible/install-chroot/*.deb 2>/dev/null | tr '\n' ',')\" 2>&1 | tee /tmp/dj_make_chroot.log" + environment: + DEBMIRROR: https://pc2cancer.ecs.csus.edu/ubuntu args: creates: "/chroot/domjudge-systest" From 2b2091c7abbec50f69f16713726f75c921191649 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Tue, 29 Jan 2019 12:30:18 +0100 Subject: [PATCH 12/17] We need the cgroups script now that we have our own VM for judgedaemons --- icpc-wf/ansible/files/judgedaemon0.service | 2 ++ icpc-wf/ansible/files/judgedaemon1.service | 2 ++ icpc-wf/ansible/files/judgedaemon2.service | 2 ++ icpc-wf/ansible/files/judgedaemon3.service | 2 ++ icpc-wf/ansible/files/judgedaemon4.service | 2 ++ icpc-wf/ansible/files/judgedaemon5.service | 2 ++ icpc-wf/ansible/judgehost.yml | 2 ++ 7 files changed, 14 insertions(+) diff --git a/icpc-wf/ansible/files/judgedaemon0.service b/icpc-wf/ansible/files/judgedaemon0.service index d9ce12e5..836d4f9c 100644 --- a/icpc-wf/ansible/files/judgedaemon0.service +++ b/icpc-wf/ansible/files/judgedaemon0.service @@ -1,6 +1,8 @@ [Unit] Description=DOMjudge JudgeDaemon After=network.target +Requires=create_cgroups.service +After=create_cgroups.service [Service] Type=simple diff --git a/icpc-wf/ansible/files/judgedaemon1.service b/icpc-wf/ansible/files/judgedaemon1.service index ee074ea2..cb8f4b78 100644 --- a/icpc-wf/ansible/files/judgedaemon1.service +++ b/icpc-wf/ansible/files/judgedaemon1.service @@ -1,6 +1,8 @@ [Unit] Description=DOMjudge JudgeDaemon After=network.target +Requires=create_cgroups.service +After=create_cgroups.service [Service] Type=simple diff --git a/icpc-wf/ansible/files/judgedaemon2.service b/icpc-wf/ansible/files/judgedaemon2.service index 552d4a7a..8582831c 100644 --- a/icpc-wf/ansible/files/judgedaemon2.service +++ b/icpc-wf/ansible/files/judgedaemon2.service @@ -1,6 +1,8 @@ [Unit] Description=DOMjudge JudgeDaemon After=network.target +Requires=create_cgroups.service +After=create_cgroups.service [Service] Type=simple diff --git a/icpc-wf/ansible/files/judgedaemon3.service b/icpc-wf/ansible/files/judgedaemon3.service index 85c4f201..72ee4637 100644 --- a/icpc-wf/ansible/files/judgedaemon3.service +++ b/icpc-wf/ansible/files/judgedaemon3.service @@ -1,6 +1,8 @@ [Unit] Description=DOMjudge JudgeDaemon After=network.target +Requires=create_cgroups.service +After=create_cgroups.service [Service] Type=simple diff --git a/icpc-wf/ansible/files/judgedaemon4.service b/icpc-wf/ansible/files/judgedaemon4.service index feb0d5d8..4d667815 100644 --- a/icpc-wf/ansible/files/judgedaemon4.service +++ b/icpc-wf/ansible/files/judgedaemon4.service @@ -1,6 +1,8 @@ [Unit] Description=DOMjudge JudgeDaemon After=network.target +Requires=create_cgroups.service +After=create_cgroups.service [Service] Type=simple diff --git a/icpc-wf/ansible/files/judgedaemon5.service b/icpc-wf/ansible/files/judgedaemon5.service index 004573ce..e2da6e91 100644 --- a/icpc-wf/ansible/files/judgedaemon5.service +++ b/icpc-wf/ansible/files/judgedaemon5.service @@ -1,6 +1,8 @@ [Unit] Description=DOMjudge JudgeDaemon After=network.target +Requires=create_cgroups.service +After=create_cgroups.service [Service] Type=simple diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index 887cca73..dae888b1 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -74,6 +74,7 @@ copy: src=files/{{item}}.service dest=/etc/systemd/system/ tags: updateservice with_items: + - create_cgroups - judgedaemon0 - judgedaemon1 - judgedaemon2 @@ -88,6 +89,7 @@ - name: enable and restart the services we just copied service: name={{item}} enabled=yes state=restarted with_items: + - create_cgroups - judgedaemon0 - judgedaemon1 - judgedaemon2 From c02130b6cf9df14fe29ddfa4e400a5c2ba7d0a94 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Tue, 29 Jan 2019 12:37:53 +0100 Subject: [PATCH 13/17] Use correct path for cgroups script --- icpc-wf/ansible/files/create_cgroups.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icpc-wf/ansible/files/create_cgroups.service b/icpc-wf/ansible/files/create_cgroups.service index 36031fba..b2f6fd8c 100644 --- a/icpc-wf/ansible/files/create_cgroups.service +++ b/icpc-wf/ansible/files/create_cgroups.service @@ -3,5 +3,5 @@ Description=Make sure cgroups exist for domjudge judgedaemon [Service] Type=oneshot -ExecStart=/home/domjudge/domjudge/bin/create_cgroups +ExecStart=/opt/domjudge-systest/bin/create_cgroups RemainAfterExit=true From 69702b70e2dd732c2fda4327fdd736f01a26bec4 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Tue, 29 Jan 2019 13:48:31 +0100 Subject: [PATCH 14/17] Use systest.domjudge.org as baseurl --- icpc-wf/ansible/common_tasks_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icpc-wf/ansible/common_tasks_build.yml b/icpc-wf/ansible/common_tasks_build.yml index 7df18be7..252e2511 100644 --- a/icpc-wf/ansible/common_tasks_build.yml +++ b/icpc-wf/ansible/common_tasks_build.yml @@ -3,7 +3,7 @@ - name: run maintainer-conf become: yes become_user: domjudge - command: make maintainer-conf CONFIGURE_FLAGS='--disable-doc-build --with-judgehost_chrootdir=/chroot/domjudge-systest' + command: make maintainer-conf CONFIGURE_FLAGS='--disable-doc-build --with-judgehost_chrootdir=/chroot/domjudge-systest --with-baseurl=https://systest.domjudge.org/' register: dj_configured args: chdir: "{{DJ_DIR}}" From d12ef0db3d363841ba9484987f3c5572c27b4a63 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Tue, 29 Jan 2019 20:36:26 +0100 Subject: [PATCH 15/17] Do not install apt-transport-https for https apt mirrors, as it does not exist anymore (cherry picked from commit b220e0f4d86cdeb69bc7183dfa3989349371878c) # Conflicts: # icpc-wf/ansible/judgehost.yml --- icpc-wf/ansible/judgehost.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index dae888b1..f6ad40d3 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -53,6 +53,15 @@ line: '\1' backrefs: yes + - name: enable internal monitor + file: path=/usr/share/X11/xorg.conf.d/22-icpc.conf state=absent + + - name: remove apt-transport-https from bionic debootstrap file + lineinfile: + path: /usr/share/debootstrap/scripts/bionic + regexp: 'ca-certificates' + line: ' base="$base ca-certificates"' + - name: copy chroot DEB packages to install copy: src=files/install-chroot dest=/tmp/dj_ansible/ From 64104c661e28545b8901a67b27d948df195eb556 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Wed, 30 Jan 2019 20:15:01 +0100 Subject: [PATCH 16/17] Fix setting php settings --- icpc-wf/ansible/domserver.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/icpc-wf/ansible/domserver.yml b/icpc-wf/ansible/domserver.yml index 0de74517..3925d3bf 100644 --- a/icpc-wf/ansible/domserver.yml +++ b/icpc-wf/ansible/domserver.yml @@ -250,11 +250,11 @@ file: state=absent path=/etc/nginx/sites-enabled/default notify: restart nginx - - name: Increase PM max children for PHP FPM + - name: Set PHP settings lineinfile: dest: /etc/php/7.2/fpm/pool.d/domjudge.conf - regexp: '^pm\.max_children' - line: 'pm.max_children = 300' + regexp: "{{item.regexp}}" + line: "{{item.key}} = {{item.value}}" with_items: - { key: 'pm.max_children', regexp: '^pm\.max_children', value: '{{PHP_FPM_MAX_CHILDREN}}' } - { key: 'php_admin_value[memory_limit]', regexp: '^php_admin_value\[memory_limit\]', value: '{{PHP_MEMORY_LIMIT}}' } From 21f771094a4e554d2301c0ef8875b373119d0a46 Mon Sep 17 00:00:00 2001 From: Nicky Gerritsen Date: Wed, 30 Jan 2019 20:49:56 +0100 Subject: [PATCH 17/17] Split out stuff for the porto host --- icpc-wf/ansible/judgehost.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/icpc-wf/ansible/judgehost.yml b/icpc-wf/ansible/judgehost.yml index f6ad40d3..ebd00633 100644 --- a/icpc-wf/ansible/judgehost.yml +++ b/icpc-wf/ansible/judgehost.yml @@ -79,10 +79,14 @@ line: 'GRUB_CMDLINE_LINUX_DEFAULT="quiet splash cgroup_enable=memory swapaccount=1"' notify: update-grub + - name: copy script to disable turboboost and hyperthreading + copy: src=files/disable-turboboost_ht dest=/usr/local/sbin/ mode=0755 + - name: copy systemd unit files copy: src=files/{{item}}.service dest=/etc/systemd/system/ tags: updateservice with_items: + - tune_cpu - create_cgroups - judgedaemon0 - judgedaemon1 @@ -95,13 +99,22 @@ - name: make sure systemctl is restarted meta: flush_handlers + - name: enable and restart the tune_cpu service + service: name=tune_cpu enabled=yes state=restarted + when: inventory_hostname == "porto-host" + - name: enable and restart the services we just copied service: name={{item}} enabled=yes state=restarted with_items: - create_cgroups - judgedaemon0 + + - name: enable and restart other judgehosts + service: name={{item}} enabled=yes state=restarted + with_items: - judgedaemon1 - judgedaemon2 - judgedaemon3 - judgedaemon4 - judgedaemon5 + when: inventory_hostname != "porto-host"