Merge branch 'feature/dockerized_build'

This commit is contained in:
Kenneth Skovhede
2017-01-14 13:18:09 +01:00
8 changed files with 307 additions and 177 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ IF NOT EXIST "%1" (
goto EXIT
)
call "%VS120COMNTOOLS%vsvars32.bat"
call "%VS140COMNTOOLS%vsvars32.bat"
7z x -oDuplicati %1
+28
View File
@@ -0,0 +1,28 @@
FROM ubuntu:latest
# Source build packages locally
ARG DOCKER_BUILD_PROXY
ENV DOCKER_BUILD_PROXY $DOCKER_BUILD_PROXY
# Install common build tools
RUN set -uex; \
export http_proxy=${HTTP_PROXY-${DOCKER_BUILD_PROXY}}; \
apt-get update; \
apt-get install --no-install-suggests --no-install-recommends -y \
build-essential \
debhelper \
dpkg-dev \
mono-devel \
libappindicator0.1-cil-dev \
ca-certificates-mono \
gtk-sharp2 \
nuget; \
apt-get clean all
RUN nuget update -self
label org.label-schema.name = "duplicati/debian-build" \
org.label-schema.version = "20161230" \
org.label-schema.vendor="Deployable" \
org.label-schema.docker.cmd="docker run -ti duplicati/debian-build" \
org.label-schema.schema-version="1.0"
+56
View File
@@ -0,0 +1,56 @@
#!/bin/bash
if [ ! -f "$1" ]; then
echo "Please provide the filename of an existing zip build as the first argument"
exit
fi
FILENAME=`basename $1`
DIRNAME=`echo "${FILENAME}" | cut -d "_" -f 1`
VERSION=`echo "${DIRNAME}" | cut -d "-" -f 2`
DATE_STAMP=`LANG=C date -R`
if [ -d "${DIRNAME}" ]; then
rm -rf "${DIRNAME}"
fi
unzip -d "${DIRNAME}" "$1"
for n in "../oem" "../../oem" "../../../oem"
do
if [ -d $n ]; then
echo "Installing OEM files"
cp -R $n "${DIRNAME}/webroot/"
fi
done
for n in "oem-app-name.txt" "oem-update-url.txt" "oem-update-key.txt" "oem-update-readme.txt" "oem-update-installid.txt"
do
for p in "../$n" "../../$n" "../../../$n"
do
if [ -f $p ]; then
echo "Installing OEM override file"
cp $p "${DIRNAME}"
fi
done
done
cp -R "debian" "${DIRNAME}"
cp "bin-rules.sh" "${DIRNAME}/debian/rules"
sed -e "s;%VERSION%;$VERSION;g" -e "s;%DATE%;$DATE_STAMP;g" "debian/changelog" > "${DIRNAME}/debian/changelog"
touch "${DIRNAME}/releasenotes.txt"
docker build -t "duplicati/debian-build:latest" - < Dockerfile.build
# Weirdness with time not being synced in Docker instance
sleep 5
docker run --workdir "/builddir/${DIRNAME}" --volume `pwd`:/builddir:rw "duplicati/debian-build:latest" dpkg-buildpackage
rm -rf "${DIRNAME}"
for filename in "duplicati_${VERSION}-1_amd64.changes" "duplicati_${VERSION}-1.dsc" "duplicati_${VERSION}-1.tar.gz"
do
if [ -f "${filename}" ]; then
rm "${filename}"
fi
done
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
DATE=`date +%Y%m%d`
VERSION=`git describe --tags | cut -d '-' -f 1 | cut -d 'v' -f 2`
GITTAG=`git rev-parse --short HEAD`
RELEASETYPE=`git describe --tags | cut -d '_' -f 2`
BUILDTAG=`git describe --tags | cut -d '-' -f 2-4`
DIRNAME="duplicati-$VERSION"
CWD=`pwd`
git pull
bash duplicati-make-git-snapshot.sh "${GITTAG}" "${DATE}" "${VERSION}" "${RELEASETYPE}" "${BUILDTAG}-${GITTAG}"
touch "${DIRNAME}/releasenotes.txt"
rm -rf "${DIRNAME}/.git"
docker build -t "duplicati/debian-build:latest" - < Dockerfile.build
# Weirdness with time not being synced in Docker instance
sleep 5
docker run --workdir "/buildroot/${DIRNAME}" --volume "${CWD}":"/buildroot":"rw" "duplicati/debian-build:latest" dpkg-buildpackage
rm -rf "${DIRNAME}"
+23
View File
@@ -0,0 +1,23 @@
FROM fedora:latest
# Source build packages locally
ARG DOCKER_BUILD_PROXY
ENV DOCKER_BUILD_PROXY $DOCKER_BUILD_PROXY
# Install common build tools
RUN dnf -y install deltarpm
RUN dnf -y upgrade
RUN dnf -y install @"Minimal Install" @buildsys-build yum-utils rpm-sign gnupg rpmdevtools
# Install mono things
RUN dnf -y install mono-devel gnome-sharp-devel dos2unix git nuget
# Fix nuget
RUN cert-sync /etc/pki/tls/certs/ca-bundle.crt
RUN nuget update -self
label org.label-schema.name = "duplicati/fedora-build" \
org.label-schema.version = "20161230" \
org.label-schema.vendor="Deployable" \
org.label-schema.docker.cmd="docker run -ti duplicati/fedora-build" \
org.label-schema.schema-version="1.0"
+82
View File
@@ -0,0 +1,82 @@
#!/bin/bash
if [ ! -f "$1" ]; then
echo "Please provide the filename of an existing zip build as the first argument"
exit
fi
FILENAME=`basename $1`
DIRNAME=`echo "${FILENAME}" | cut -d "_" -f 1`
VERSION=`echo "${DIRNAME}" | cut -d "-" -f 2`
BUILDDATE=`date +%Y%m%d`
BUILDTAG_RAW=`echo "${FILENAME}" | cut -d "." -f 1-4 | cut -d "-" -f 2-4`
BUILDTAG="${BUILDTAG_RAW//-}"
CWD=`pwd`
echo "BUILDTAG: ${BUILDTAG}"
echo "Version: ${VERSION}"
echo "Builddate: ${BUILDDATE}"
echo "Dirname: ${DIRNAME}"
#DIRNAME="duplicati-${BUILDDATE}"
if [ -d "${DIRNAME}" ]; then
rm -rf "${DIRNAME}"
fi
unzip -q -d "${DIRNAME}" "$1"
cp ../debian/*-launcher.sh "${DIRNAME}"
cp ../debian/duplicati.png "${DIRNAME}"
cp ../debian/duplicati.desktop "${DIRNAME}"
for n in "../oem" "../../oem" "../../../oem"
do
if [ -d $n ]; then
echo "Installing OEM files"
cp -R $n "${DIRNAME}/webroot/"
fi
done
for n in "oem-app-name.txt" "oem-update-url.txt" "oem-update-key.txt" "oem-update-readme.txt" "oem-update-installid.txt"
do
for p in "../$n" "../../$n" "../../../$n"
do
if [ -f $p ]; then
echo "Installing OEM override file"
cp $p "${DIRNAME}"
fi
done
done
tar -cjf "${DIRNAME}.tar.bz2" "${DIRNAME}"
rm -rf "${DIRNAME}"
RPMBUILD="${CWD}/${DIRNAME}-rpmbuild"
if [ -d "${RPMBUILD}" ]; then
rm -rf "${RPMBUILD}"
fi
mkdir -p "${RPMBUILD}"/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
mv "${DIRNAME}.tar.bz2" "${RPMBUILD}/SOURCES/"
cp duplicati.xpm "${RPMBUILD}/SOURCES/"
cp make-binary-package.sh "${RPMBUILD}/SOURCES/duplicati-make-binary-package.sh"
cp duplicati-install-recursive.sh "${RPMBUILD}/SOURCES/duplicati-install-recursive.sh"
echo "%global _builddate ${BUILDDATE}" > "${RPMBUILD}/SOURCES/duplicati-buildinfo.spec"
echo "%global _buildversion ${VERSION}" >> "${RPMBUILD}/SOURCES/duplicati-buildinfo.spec"
echo "%global _buildtag ${BUILDTAG}" >> "${RPMBUILD}/SOURCES/duplicati-buildinfo.spec"
docker build -t "duplicati/fedora-build:latest" - < Dockerfile.build
# Weirdness with time not being synced in Docker instance
sleep 5
docker run \
--workdir "/buildroot" \
--volume "${CWD}":"/buildroot":"rw" \
--volume "${RPMBUILD}":"/root/rpmbuild":"rw" \
"duplicati/fedora-build:latest" \
rpmbuild -bb duplicati-binary.spec
mv "${RPMBUILD}/RPMS/noarch/"*.rpm .
rm -rf "${RPMBUILD}"
+52
View File
@@ -0,0 +1,52 @@
#!/bin/bash
git pull
DATE=`date +%Y%m%d`
VERSION=`git describe --tags | cut -d '-' -f 1 | cut -d 'v' -f 2`
GITTAG=`git rev-parse --short HEAD`
RELEASETYPE=`git describe --tags | cut -d '_' -f 2`
BUILDTAG=`git describe --tags | cut -d '-' -f 2-4`
CWD=`pwd`
bash duplicati-make-git-snapshot.sh "${GITTAG}" "${DATE}" "${VERSION}" "${RELEASETYPE}" "${BUILDTAG}-${GITTAG}"
RPMBUILD="${CWD}/${BUILDTAG}-rpmbuild"
if [ -d "${RPMBUILD}" ]; then
rm -rf "${RPMBUILD}"
fi
mkdir -p "${RPMBUILD}"/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
mv duplicati-$DATE.tar.bz2 "${RPMBUILD}/SOURCES/"
cp *.sh "${RPMBUILD}/SOURCES/"
cp *.patch "${RPMBUILD}/SOURCES/"
cp duplicati.xpm "${RPMBUILD}/SOURCES/"
cp build-package.sh "${RPMBUILD}/SOURCES/duplicati-build-package.sh"
echo "%global _gittag ${GITTAG}" > "${RPMBUILD}/SOURCES/duplicati-buildinfo.spec"
echo "%global _builddate ${DATE}" >> "${RPMBUILD}/SOURCES/duplicati-buildinfo.spec"
echo "%global _buildversion ${VERSION}" >> "${RPMBUILD}/SOURCES/duplicati-buildinfo.spec"
echo "%global _releasetype ${RELEASETYPE}" >> "${RPMBUILD}/SOURCES/duplicati-buildinfo.spec"
docker build -t "duplicati/fedora-build:latest" - < Dockerfile.build
# Weirdness with time not being synced in Docker instance
sleep 5
docker run \
--workdir "/buildroot" \
--volume "${CWD}":"/buildroot":"rw" \
--volume "${RPMBUILD}":"/root/rpmbuild":"rw" \
"duplicati/fedora-build:latest" \
rpmbuild -bs duplicati.spec
docker run \
--workdir "/buildroot" \
--volume "${CWD}":"/buildroot":"rw" \
--volume "${RPMBUILD}":"/root/rpmbuild":"rw" \
"duplicati/fedora-build:latest" \
rpmbuild -bb duplicati.spec
mv "${RPMBUILD}/RPMS/noarch/"*.rpm .
mv "${RPMBUILD}/SRPMS/"*.rpm .
rm -rf "${RPMBUILD}"
+41 -176
View File
@@ -6,6 +6,18 @@ then
exit 1
fi
while true
do
DOCKER_RESULT=$(docker ps)
if [ "$?" != "0" ]
then
echo "It appears the Docker daemon is not running, make sure you started it"
read -p "Press [Enter] key to AFTER you started Docker"
continue
fi
break
done
GITHUB_TOKEN_FILE="${HOME}/.config/github-api-token"
GPG_KEYFILE="${HOME}/.config/signkeys/Duplicati/updater-gpgkey.key"
AUTHENTICODE_PFXFILE="${HOME}/.config/signkeys/Duplicati/authenticode.pfx"
@@ -14,15 +26,6 @@ MONO=/Library/Frameworks/Mono.framework/Commands/mono
GPG=/usr/local/bin/gpg2
FEDORA_INSTANCE_ID=i-deef5352
FEDORA_PUBKEY=AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM3OWpUJOqoh9hq/k48g/FFLqnxUHxecVZM/jRD69Y/cn0OygsSyi3E5X/PVgtfyoced/HV788f9rDpLbY08jXg=
DEBIAN_INSTANCE_ID=i-f237887e
DEBIAN_PUBKEY=AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJUDOPM7lBBOUtRbAZ8DUP7hHN88CvSvKzWQ15uXcOhdMgtqlznSQiLNfagD/AlqnPsOXWCnNN0fM+QAeMuMBEY=
WINDOWS_INSTANCE_ID=i-be348b32
WINDOWS_PUBKEY=AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK5MD6Jg+p6rJmJfx8lWqIY+ZLDEYcGIwcf+eGzPSz4QKuOgxtDo0xgg3/OrxJibyk7sBWkT0f9RHD1qN9Nz/o8=
ZIPFILE=`basename "$1"`
VERSION=`echo "${ZIPFILE}" | cut -d "-" -f 2 | cut -d "_" -f 1`
BUILDTYPE=`echo "${ZIPFILE}" | cut -d "-" -f 2 | cut -d "_" -f 2`
@@ -40,9 +43,6 @@ SIGNAME="duplicati-${BUILDTAG_RAW}-signatures.zip"
UPDATE_TARGET="Updates/build/${BUILDTYPE}_target-${VERSION}"
KEYFILE=~/.ssh/duplicati-build-machines-key.pem
SSH_OPTIONS="-i ${KEYFILE} -o UserKnownHostsFile=./tmp/known_hosts"
echo "Filename: ${ZIPFILE}"
echo "Version: ${VERSION}"
echo "Buildtype: ${BUILDTYPE}"
@@ -51,79 +51,6 @@ echo "RPMName: ${RPMNAME}"
echo "DEBName: ${DEBNAME}"
echo "SPKName: ${SPKNAME}"
start_aws_instance() {
local STATE=`aws ec2 describe-instances --profile=duplicati-builder --instance-ids $1 | python -c 'import sys, json; print json.load(sys.stdin)["Reservations"][0]["Instances"][0]["State"]["Name"]'`
if [ "${STATE}" == "stopping" ]; then
echo -n "Instance has state '${STATE}', waiting .."
while [ "${STATE}" == "stopping" ]; do
echo -n "."
sleep 5
local STATE=`aws ec2 describe-instances --profile=duplicati-builder --instance-ids $1 | python -c 'import sys, json; print json.load(sys.stdin)["Reservations"][0]["Instances"][0]["State"]["Name"]'`
done
echo ""
fi
local STATE=`aws ec2 start-instances --profile=duplicati-builder --instance-ids $1 | python -c 'import sys, json; print json.load(sys.stdin)["StartingInstances"][0]["CurrentState"]["Name"]'`
if [ "${STATE}" != "running" ]; then
echo -n "Instance has state '${STATE}', waiting .."
while [ "${STATE}" != "running" ]; do
echo -n "."
sleep 5
local STATE=`aws ec2 describe-instances --profile=duplicati-builder --instance-ids $1 | python -c 'import sys, json; print json.load(sys.stdin)["Reservations"][0]["Instances"][0]["State"]["Name"]'`
done
echo ""
fi
}
stop_aws_instance() {
local STATE=`aws ec2 stop-instances --profile=duplicati-builder --instance-ids $1 | python -c 'import sys, json; print json.load(sys.stdin)["StoppingInstances"][0]["CurrentState"]["Name"]'`
echo "Instance state is now: ${STATE}"
}
ssh_connect_to_instance() {
local DNSNAME=`aws ec2 describe-instances --profile=duplicati-builder --instance-ids $1 | python -c 'import sys, json; print json.load(sys.stdin)["Reservations"][0]["Instances"][0]["PublicDnsName"]'`
local IPADDR=`aws ec2 describe-instances --profile=duplicati-builder --instance-ids $1 | python -c 'import sys, json; print json.load(sys.stdin)["Reservations"][0]["Instances"][0]["PublicIpAddress"]'`
if [ -d "./tmp" ]; then
rm -rf "./tmp"
fi
mkdir "./tmp"
echo "${DNSNAME},${IPADDR} ecdsa-sha2-nistp256 $3" > "./tmp/known_hosts"
SSH_HOST="$2@${DNSNAME}"
ssh ${SSH_OPTIONS} "${SSH_HOST}" "cd" > /dev/null
local EXITCODE=$?
if [ ! "${EXITCODE}" -eq 0 ]; then
echo -n "Server did not allow login, waiting .."
while [ ! "${EXITCODE}" -eq 0 ]; do
echo -n "."
sleep 5
ssh ${SSH_OPTIONS} "${SSH_HOST}" "cd" > /dev/null
local EXITCODE=$?
done
echo ""
fi
}
ssh_upload_file() {
scp ${SSH_OPTIONS} "$1" "${SSH_HOST}:"
}
ssh_run_commands() {
cat "$1" | ssh ${SSH_OPTIONS} "${SSH_HOST}"
}
build_file_signatures() {
if [ "z${GPGID}" != "z" ]; then
echo "$GPGKEY" | "${GPG}" "--passphrase-fd" "0" "--batch" "--yes" "--default-key=${GPGID}" "--output" "$2.sig" "--detach-sig" "$1"
@@ -153,11 +80,10 @@ else
echo "No GPG keyfile found, skipping gpg signatures"
fi
# Pre-boot instances to keep the waiting to a minimun
# Pre-boot virtual machine
echo "Booting Win10 build instance"
VBoxHeadless --startvm Duplicati-Win10-Build &
aws ec2 start-instances --profile=duplicati-builder --instance-ids "${DEBIAN_INSTANCE_ID}" &> /dev/null
aws ec2 start-instances --profile=duplicati-builder --instance-ids "${FEDORA_INSTANCE_ID}" &> /dev/null
aws ec2 start-instances --profile=duplicati-builder --instance-ids "${WINDOWS_INSTANCE_ID}" &> /dev/null
# Then do the local build to mask the waiting a little more
@@ -167,11 +93,11 @@ echo "Building OSX package locally ..."
echo ""
echo "Enter local sudo password..."
cd Installer/OSX
cd "Installer/OSX"
bash "make-dmg.sh" "../../$1"
mv "Duplicati.dmg" "../../${UPDATE_TARGET}/${DMGNAME}"
mv "Duplicati.pkg" "../../${UPDATE_TARGET}/${PKGNAME}"
cd ../..
cd "../.."
echo ""
echo ""
@@ -185,112 +111,48 @@ cd ../..
echo ""
echo ""
echo "Starting Debian build instance"
echo "Building Debian deb with Docker ..."
start_aws_instance "${DEBIAN_INSTANCE_ID}"
cd "Installer/debian"
bash "docker-build-binary.sh" "../../$1"
cd "../.."
ssh_connect_to_instance "${DEBIAN_INSTANCE_ID}" "ubuntu" "${DEBIAN_PUBKEY}"
echo "Instance has started, uploading binary package ..."
ssh_upload_file "$1"
echo "Running build script on server ..."
cat > "./tmp/debian-commands.sh" <<EOF
cd duplicati/Installer/debian
git pull
bash make-binary-package.sh "../../../${ZIPFILE}"
rm -rf "../../../${ZIPFILE}"
EOF
ssh_run_commands "./tmp/debian-commands.sh"
echo "Downloading binary package ..."
scp ${SSH_OPTIONS} "${SSH_HOST}:duplicati/Installer/debian/${DEBNAME}" "./tmp"
ssh ${SSH_OPTIONS} "${SSH_HOST}" "rm -rf duplicati/Installer/debian/duplicati_${VERSION}-*"
echo "Done, stopping instance ..."
stop_aws_instance "${DEBIAN_INSTANCE_ID}"
mv "./tmp/${DEBNAME}" "${UPDATE_TARGET}/"
mv "Installer/debian/${DEBNAME}" "${UPDATE_TARGET}"
echo "Done building deb package"
echo ""
echo ""
echo "Starting Fedora build instance ..."
echo "Building Fedora RPM with Docker ..."
start_aws_instance "${FEDORA_INSTANCE_ID}"
cd "Installer/fedora"
bash "docker-build-binary.sh" "../../$1"
cd "../.."
ssh_connect_to_instance "${FEDORA_INSTANCE_ID}" "ec2-user" "${FEDORA_PUBKEY}"
mv "Installer/fedora/${RPMNAME}" "${UPDATE_TARGET}"
echo "Instance has started, uploading binary package ..."
ssh_upload_file "$1"
echo "Running build script on server ..."
cat > "./tmp/fedora-commands.sh" <<EOF
rm -rf ~/rpmbuild
rpmdev-setuptree
cd duplicati/Installer/fedora
git pull
bash make-binary-package.sh "../../../${ZIPFILE}"
rm -rf "../../../${ZIPFILE}"
EOF
ssh_run_commands "./tmp/fedora-commands.sh"
echo "Downloading binary package ..."
scp ${SSH_OPTIONS} "${SSH_HOST}:rpmbuild/RPMS/noarch/${RPMNAME}" "./tmp"
echo "Done, stopping instance ..."
stop_aws_instance "${FEDORA_INSTANCE_ID}"
mv "./tmp/${RPMNAME}" "${UPDATE_TARGET}/"
echo "Done building rpm package"
echo ""
echo ""
echo "Starting Windows build instance"
echo "Building Windows instance in virtual machine"
start_aws_instance "${WINDOWS_INSTANCE_ID}"
ssh_connect_to_instance "${WINDOWS_INSTANCE_ID}" "ec2-user" "${WINDOWS_PUBKEY}"
echo "Instance has started, uploading binary package ..."
ssh_upload_file "$1"
echo "Running build script on server ..."
cat > "./tmp/windows-commands.sh" <<EOF
cat > "tmp-windows-commands.bat" <<EOF
SET VS120COMNTOOLS=%VS140COMNTOOLS%
cd duplicati\\Installer\\Windows
git pull
build-msi.bat "..\\..\\..\\${ZIPFILE}"
del /q "..\\..\\..\\${ZIPFILE}"
cd \\Duplicati\\Installer\\Windows
build-msi.bat "../../$1"
EOF
ssh_run_commands "./tmp/windows-commands.sh"
ssh IEUser@192.168.56.101 "\\Duplicati\\tmp-windows-commands.bat"
ssh IEUser@192.168.56.101 "shutdown /s /t 0"
echo "Downloading binary packages ..."
rm "tmp-windows-commands.bat"
scp ${SSH_OPTIONS} "${SSH_HOST}:duplicati/Installer/Windows/Duplicati.msi" "./tmp"
scp ${SSH_OPTIONS} "${SSH_HOST}:duplicati/Installer/Windows/Duplicati-32bit.msi" "./tmp"
echo "Done, stopping instance ..."
stop_aws_instance "${WINDOWS_INSTANCE_ID}"
mv "./tmp/Duplicati.msi" "${UPDATE_TARGET}/${MSI64NAME}"
mv "./tmp/Duplicati-32bit.msi" "${UPDATE_TARGET}/${MSI32NAME}"
mv "./Installer/Windows/Duplicati.msi" "${UPDATE_TARGET}/${MSI64NAME}"
mv "./Installer/Windows/Duplicati-32bit.msi" "${UPDATE_TARGET}/${MSI32NAME}"
if [ -f "${AUTHENTICODE_PFXFILE}" ] && [ -f "${AUTHENTICODE_PASSWORD}" ]; then
echo "Performing authenticode signing of installers"
@@ -432,3 +294,6 @@ if [ -f ~/.config/duplicati-mirror-sync.sh ]; then
else
echo "Skipping CDN update"
fi
VBoxManage controlvm "Duplicati-Win10-Build" poweroff