From 37e801e39ee34a9a6d55287ce07481a892f8fd94 Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Mon, 15 Jul 2019 10:36:12 +0900 Subject: [PATCH 1/5] Tidy up build scripts --- .travis.yml | 58 ++++------------------------------ scripts/before_install_3.7 | 30 ++++++++++++++++++ scripts/renew-catalog-template | 31 ++++++++++++++++++ scripts/upload-catalog | 24 ++++++++++++++ 4 files changed, 92 insertions(+), 51 deletions(-) create mode 100644 scripts/before_install_3.7 create mode 100644 scripts/renew-catalog-template create mode 100644 scripts/upload-catalog diff --git a/.travis.yml b/.travis.yml index ce10a5698..8afd630c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ -dist: trusty +dist: xenial language: python -python: '3.6' +python: '3.7' branches: only: - 3.7 @@ -14,61 +14,17 @@ env: - CATALOG_BRANCH=catalog-3.7 # Transifex project name - TRANSIFEX_PROJECT=python-newest + # Directory where repositories are cloned + - BASEDIR="$(dirname ${TRAVIS_BUILD_DIR})" before_install: -- export TZ=Asia/Tokyo -- git checkout ${DOCS_BRANCH} -- cd .. -- pwd -- ls -la -- BASEDIR="$(pwd)" -- cd ~ -- openssl aes-256-cbc -K ${encryption_37_K} -iv ${encryption_37_iv} -in "${BASEDIR}/python-docs-ja/secrets_${DOCS_BRANCH}.tar.enc" -out ~/secrets_${DOCS_BRANCH}.tar -d -- tar xvf ~/secrets_${DOCS_BRANCH}.tar -- rm ~/secrets_${DOCS_BRANCH}.tar -- chmod 600 ~/.ssh/python-docs-ja_${DOCS_BRANCH} -- chmod 600 ~/.ssh/cpython-doc-catalog_${CATALOG_BRANCH} -# Hack from: https://gist.github.com/jamesmcfadden/d379e04e7ae2861414886af189ec59e5 -- echo -e "Host python-docs-ja.github.com\n\tHostName github.com\n\tIdentityFile ~/.ssh/python-docs-ja_${DOCS_BRANCH}\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config -- echo -e "Host cpython-doc-catalog.github.com\n\tHostName github.com\n\tIdentityFile ~/.ssh/cpython-doc-catalog_${CATALOG_BRANCH}\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config -- cat ~/.ssh/config -- git config --global user.email "travisci-build-bot@example.com" -- git config --global user.name "Autobuild bot on TravisCI" +- bash ./scripts/before_install_3.7 install: - pip install sphinx - pip install blurb - pip install transifex-client - pip install sphinx-intl -- cd "${BASEDIR}" -- git clone --depth 50 --branch ${CATALOG_BRANCH} https://github.com/python-doc-ja/cpython-doc-catalog.git cpython-doc-catalog -- mkdir -p "${BASEDIR}"/cpython-doc-catalog/Doc/locales/ja -- cd "${BASEDIR}"/cpython-doc-catalog/Doc/locales/ja -- ln -s "${BASEDIR}"/python-docs-ja LC_MESSAGES -- ls -lF LC_MESSAGES script: # upload-catalog -- cd "${BASEDIR}"/cpython-doc-catalog/Doc/locales -- tx pull --force --language ja -- cd ja/LC_MESSAGES -- git add *.po **/*.po -- git status -- if [ $(git status --short | wc -l) -eq 0 ]; then echo "no .po file to upload"; else echo "I have .po file(s) to upload"; git commit --message="[skip ci] Update .po files"; git push --quiet "git@python-docs-ja.github.com:python/python-docs-ja.git" ${DOCS_BRANCH}:${DOCS_BRANCH}; fi -- rm -rf "${BASEDIR}"/cpython-doc-catalog/Doc/locales/ja +- bash ./scripts/upload-catalog # renew-catalog-template -- cd "${BASEDIR}"/cpython-doc-catalog -- git remote add upstream https://github.com/python/cpython.git -- git remote -v -- git fetch --quiet upstream -- git merge --no-ff upstream/${CPYTHON_BRANCH} -m "Merge remote-tracking branch 'upstream/${CPYTHON_BRANCH}' into ${CATALOG_BRANCH} by Autobuild bot on TravisCI" -- cd Doc -- make build ALLSPHINXOPTS="-E -b gettext -D gettext_compact=0 -d build/.doctrees . locales/pot" -- ls -lt locales/pot -- cd locales -- git add pot -- if [ $(git status -s | wc -l) -eq 0 ]; then echo "no .pot file to update"; exit 0; else echo "I have .pot file(s) to upload"; fi -- rm -rf .tx -- sphinx-intl create-txconfig -- sphinx-intl update-txconfig-resources --transifex-project-name=${TRANSIFEX_PROJECT} --locale-dir . --pot-dir pot -- tx push -s -- git add .tx -- git commit --message="[skip ci] Update .pot files and .tx/config" -- git push --quiet "git@cpython-doc-catalog.github.com:python-doc-ja/cpython-doc-catalog.git" ${CATALOG_BRANCH}:${CATALOG_BRANCH} +- bash ./scripts/renew-catalog-template diff --git a/scripts/before_install_3.7 b/scripts/before_install_3.7 new file mode 100644 index 000000000..624e8e229 --- /dev/null +++ b/scripts/before_install_3.7 @@ -0,0 +1,30 @@ +#!/bin/bash + +ls -la ${BASEDIR} +git config --global user.email "travisci-build-bot@example.com" +git config --global user.name "Autobuild bot on TravisCI" +git checkout ${DOCS_BRANCH} + +# extract secrets +cd ~ +openssl aes-256-cbc -K ${encryption_37_K} -iv ${encryption_37_iv} -in "${BASEDIR}/python-docs-ja/secrets_${DOCS_BRANCH}.tar.enc" -out ~/secrets_${DOCS_BRANCH}.tar -d +tar xvf ~/secrets_${DOCS_BRANCH}.tar +rm ~/secrets_${DOCS_BRANCH}.tar +chmod 600 ~/.ssh/python-docs-ja_${DOCS_BRANCH} +chmod 600 ~/.ssh/cpython-doc-catalog_${CATALOG_BRANCH} + +# enable using plural deploy keys for repositories +# Hack from: https://gist.github.com/jamesmcfadden/d379e04e7ae2861414886af189ec59e5 +cat <> ~/.ssh/config +Host python-docs-ja.github.com + HostName github.com + IdentityFile ~/.ssh/python-docs-ja_${DOCS_BRANCH} + StrictHostKeyChecking no + +Host cpython-doc-catalog.github.com + HostName github.com + IdentityFile ~/.ssh/cpython-doc-catalog_${CATALOG_BRANCH} + StrictHostKeyChecking no +EOF + +cat ~/.ssh/config diff --git a/scripts/renew-catalog-template b/scripts/renew-catalog-template new file mode 100644 index 000000000..181130cdb --- /dev/null +++ b/scripts/renew-catalog-template @@ -0,0 +1,31 @@ +#!/bin/bash + +# merge from upstream +cd "${BASEDIR}"/cpython-doc-catalog +git remote add upstream https://github.com/python/cpython.git +git remote -v +git fetch --quiet upstream +git merge --no-ff upstream/${CPYTHON_BRANCH} -m "Merge remote-tracking branch 'upstream/${CPYTHON_BRANCH}' into ${CATALOG_BRANCH} by Autobuild bot on TravisCI" + +# generate catalog +cd Doc +make build ALLSPHINXOPTS="-E -b gettext -D gettext_compact=0 -d build/.doctrees . locales/pot" +ls -lt locales/pot + +# upload catalog templates to cpython-doc-catalog +cd locales +git add pot +if [ $(git status -s | wc -l) -eq 0 ]; then + echo "no .pot file to update" + exit 0 +fi + +echo "I have .pot file(s) to upload" + +rm -rf .tx +sphinx-intl create-txconfig +sphinx-intl update-txconfig-resources --transifex-project-name=${TRANSIFEX_PROJECT} --locale-dir . --pot-dir pot +tx push -s +git add .tx +git commit --message="[skip ci] Update .pot files and .tx/config" +git push --quiet "git@cpython-doc-catalog.github.com:python-doc-ja/cpython-doc-catalog.git" ${CATALOG_BRANCH}:${CATALOG_BRANCH} diff --git a/scripts/upload-catalog b/scripts/upload-catalog new file mode 100644 index 000000000..9e42b0c84 --- /dev/null +++ b/scripts/upload-catalog @@ -0,0 +1,24 @@ +#!/bin/bash + +# clone an additional repository +cd "${BASEDIR}" +git clone --depth 50 --branch ${CATALOG_BRANCH} https://github.com/python-doc-ja/cpython-doc-catalog.git cpython-doc-catalog +mkdir -p "${BASEDIR}"/cpython-doc-catalog/Doc/locales/ja +cd "${BASEDIR}"/cpython-doc-catalog/Doc/locales/ja +ln -s "${BASEDIR}"/python-docs-ja LC_MESSAGES +ls -lF LC_MESSAGES + +# upload catalogs to python-docs-ja +cd "${BASEDIR}"/cpython-doc-catalog/Doc/locales +tx pull --force --language ja +cd ja/LC_MESSAGES +git add *.po **/*.po +git status +if [ $(git status --short | wc -l) -eq 0 ]; then + echo "no .po file to upload" +else + echo "I have .po file(s) to upload" + git commit --message="[skip ci] Update .po files" + git push --quiet "git@python-docs-ja.github.com:python/python-docs-ja.git" ${DOCS_BRANCH}:${DOCS_BRANCH} +fi +rm -rf "${BASEDIR}"/cpython-doc-catalog/Doc/locales/ja From b23156ea09fc335492b43be1fd2c58083baa3886 Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Tue, 16 Jul 2019 02:45:26 +0900 Subject: [PATCH 2/5] Make log more verbose --- scripts/before_install_3.7 | 1 + scripts/renew-catalog-template | 1 + scripts/upload-catalog | 1 + 3 files changed, 3 insertions(+) diff --git a/scripts/before_install_3.7 b/scripts/before_install_3.7 index 624e8e229..c0b650ab0 100644 --- a/scripts/before_install_3.7 +++ b/scripts/before_install_3.7 @@ -1,4 +1,5 @@ #!/bin/bash +set -eux ls -la ${BASEDIR} git config --global user.email "travisci-build-bot@example.com" diff --git a/scripts/renew-catalog-template b/scripts/renew-catalog-template index 181130cdb..62115f7f5 100644 --- a/scripts/renew-catalog-template +++ b/scripts/renew-catalog-template @@ -1,4 +1,5 @@ #!/bin/bash +set -eux # merge from upstream cd "${BASEDIR}"/cpython-doc-catalog diff --git a/scripts/upload-catalog b/scripts/upload-catalog index 9e42b0c84..602f2c2b8 100644 --- a/scripts/upload-catalog +++ b/scripts/upload-catalog @@ -1,4 +1,5 @@ #!/bin/bash +set -eux # clone an additional repository cd "${BASEDIR}" From e43f1bbb2a21dea4f885e2a0fad9039ab3d7425c Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Tue, 16 Jul 2019 02:48:32 +0900 Subject: [PATCH 3/5] Specify paths absolutely --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8afd630c4..801247a33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ env: # Directory where repositories are cloned - BASEDIR="$(dirname ${TRAVIS_BUILD_DIR})" before_install: -- bash ./scripts/before_install_3.7 +- bash ${TRAVIS_BUILD_DIR}/scripts/before_install_3.7 install: - pip install sphinx - pip install blurb @@ -25,6 +25,6 @@ install: - pip install sphinx-intl script: # upload-catalog -- bash ./scripts/upload-catalog +- bash ${TRAVIS_BUILD_DIR}/scripts/upload-catalog # renew-catalog-template -- bash ./scripts/renew-catalog-template +- bash ${TRAVIS_BUILD_DIR}/scripts/renew-catalog-template From f0ff5c63d97d6d528273a93ec5c0a3824673b37e Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Tue, 16 Jul 2019 03:06:17 +0900 Subject: [PATCH 4/5] [WIP] Display a branch name --- scripts/before_install_3.7 | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/before_install_3.7 b/scripts/before_install_3.7 index c0b650ab0..fcf6cf5d8 100644 --- a/scripts/before_install_3.7 +++ b/scripts/before_install_3.7 @@ -4,6 +4,7 @@ set -eux ls -la ${BASEDIR} git config --global user.email "travisci-build-bot@example.com" git config --global user.name "Autobuild bot on TravisCI" +git branch git checkout ${DOCS_BRANCH} # extract secrets From f39e9e917038bd49f5bad398948e2153035e65df Mon Sep 17 00:00:00 2001 From: cocoatomo Date: Tue, 16 Jul 2019 03:12:29 +0900 Subject: [PATCH 5/5] Revert "[WIP] Display a branch name" This reverts commit f0ff5c63d97d6d528273a93ec5c0a3824673b37e. --- scripts/before_install_3.7 | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/before_install_3.7 b/scripts/before_install_3.7 index fcf6cf5d8..c0b650ab0 100644 --- a/scripts/before_install_3.7 +++ b/scripts/before_install_3.7 @@ -4,7 +4,6 @@ set -eux ls -la ${BASEDIR} git config --global user.email "travisci-build-bot@example.com" git config --global user.name "Autobuild bot on TravisCI" -git branch git checkout ${DOCS_BRANCH} # extract secrets