setup/package.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 ##############################################################################
   4 #
   5 #    OpenERP, Open Source Management Solution
   6 #    Copyright (C) 2004-Today OpenERP SA (<http://www.openerp.com>).
   7 #
   8 #    This program is free software: you can redistribute it and/or modify
   9 #    it under the terms of the GNU Affero General Public License as
  10 #    published by the Free Software Foundation, either version 3 of the
  11 #    License, or (at your option) any later version.
  12 #
  13 #    This program is distributed in the hope that it will be useful,
  14 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #    GNU Affero General Public License for more details.
  17 #
  18 #    You should have received a copy of the GNU Affero General Public License
  19 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20 #
  21 ##############################################################################
  22
  23 import optparse
  24 import os
  25 import pexpect
  26 import shutil
  27 import signal
  28 import subprocess
  29 import tempfile
  30 import time
  31 import xmlrpclib
  32 from contextlib import contextmanager
  33 from glob import glob
  34 from os.path import abspath, dirname, join
  35 from tempfile import NamedTemporaryFile
  36
  37
  38 #----------------------------------------------------------
  39 # Utils
  40 #----------------------------------------------------------
  41 execfile(join(dirname(__file__), '..', 'openerp', 'release.py'))
  42 version = version.split('-')[0]
  43
  44 timestamp = time.strftime("%Y%m%d", time.gmtime())
  45 PUBLISH_DIRS = {
  46     'debian': 'deb',
  47     'redhat': 'rpm',
  48     'tarball': 'src',
  49     'windows': 'exe',
  50 }
  51 EXTENSIONS = [
  52     '.tar.gz',
  53     '.deb',
  54     '.dsc',
  55     '.changes',
  56     '.noarch.rpm',
  57     '.exe',
  58 ]
  59
  60 def mkdir(d):
  61     if not os.path.isdir(d):
  62         os.makedirs(d)
  63
  64 def system(l, chdir=None):
  65     print l
  66     if chdir:
  67         cwd = os.getcwd()
  68         os.chdir(chdir)
  69     if isinstance(l, list):
  70         rc = os.spawnvp(os.P_WAIT, l[0], l)
  71     elif isinstance(l, str):
  72         tmp = ['sh', '-c', l]
  73         rc = os.spawnvp(os.P_WAIT, tmp[0], tmp)
  74     if chdir:
  75         os.chdir(cwd)
  76     return rc
  77
  78 def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
  79     time.sleep(5)
  80     modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  81         dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
  82     )
  83     if modules and len(modules) > 1:
  84         time.sleep(1)
  85         toinstallmodules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  86             dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
  87         )
  88         if toinstallmodules:
  89             print("Package test: FAILED. Not able to install dependencies of base.")
  90             raise Exception("Installation of package failed")
  91         else:
  92             print("Package test: successfuly installed %s modules" % len(modules))
  93     else:
  94         print("Package test: FAILED. Not able to install base.")
  95         raise Exception("Installation of package failed")
  96
  97 def publish(o, type, releases):
  98     def _publish(o, release):
  99         arch = ''
 100         filename = release.split(os.path.sep)[-1]
 101
 102         extension = None
 103         for EXTENSION in EXTENSIONS:
 104             if filename.endswith(EXTENSION):
 105                 extension = EXTENSION
 106                 filename = filename.replace(extension, '')
 107                 break
 108         if extension is None:
 109             raise Exception("Extension of %s is not handled" % filename)
 110
 111         # keep _all or _amd64
 112         if filename.count('_') > 1:
 113             arch = '_' + filename.split('_')[-1]
 114
 115         release_dir = PUBLISH_DIRS[type]
 116         release_filename = 'odoo_%s.%s%s%s' % (version, timestamp, arch, extension)
 117         release_path = join(o.pub, release_dir, release_filename)
 118
 119         system('mkdir -p %s' % join(o.pub, release_dir))
 120         shutil.move(join(o.build_dir, release), release_path)
 121
 122         # Latest/symlink handler
 123         release_abspath = abspath(release_path)
 124         latest_abspath = release_abspath.replace(timestamp, 'latest')
 125
 126         if os.path.islink(latest_abspath):
 127             os.unlink(latest_abspath)
 128
 129         os.symlink(release_abspath, latest_abspath)
 130
 131         return release_path
 132
 133     published = []
 134     if isinstance(releases, basestring):
 135         published.append(_publish(o, releases))
 136     elif isinstance(releases, list):
 137         for release in releases:
 138             published.append(_publish(o, release))
 139     return published
 140
 141 class OdooDocker(object):
 142     def __init__(self):
 143         self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
 144         self.port = 8069  # TODO sle: reliable way to get a free port?
 145         self.prompt_re = '(\r\nroot@|bash-).*# '
 146         self.timeout = 600
 147
 148     def system(self, command):
 149         self.docker.sendline(command)
 150         self.docker.expect(self.prompt_re)
 151
 152     def start(self, docker_image, build_dir, pub_dir):
 153         self.build_dir = build_dir
 154         self.pub_dir = pub_dir
 155
 156         self.docker = pexpect.spawn(
 157             'docker run -v %s:/opt/release -p 127.0.0.1:%s:8069'
 158             ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
 159             timeout=self.timeout
 160         )
 161         time.sleep(2)  # let the bash start
 162         self.docker.logfile_read = self.log_file
 163         self.id = subprocess.check_output('docker ps -l -q', shell=True)
 164
 165     def end(self):
 166         try:
 167             _rpc_count_modules(port=str(self.port))
 168         except Exception, e:
 169             print('Exception during docker execution: %s:' % str(e))
 170             print('Error during docker execution: printing the bash output:')
 171             with open(self.log_file.name) as f:
 172                 print '\n'.join(f.readlines())
 173             raise
 174         finally:
 175             self.docker.close()
 176             system('docker rm -f %s' % self.id)
 177             self.log_file.close()
 178             os.remove(self.log_file.name)
 179
 180 @contextmanager
 181 def docker(docker_image, build_dir, pub_dir):
 182     _docker = OdooDocker()
 183     try:
 184         _docker.start(docker_image, build_dir, pub_dir)
 185         try:
 186             yield _docker
 187         except Exception, e:
 188             raise
 189     finally:
 190         _docker.end()
 191
 192 class KVM(object):
 193     def __init__(self, o, image, ssh_key='', login='openerp'):
 194         self.o = o
 195         self.image = image
 196         self.ssh_key = ssh_key
 197         self.login = login
 198
 199     def timeout(self,signum,frame):
 200         print "vm timeout kill",self.pid
 201         os.kill(self.pid,15)
 202
 203     def start(self):
 204         l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -drive".split(" ")
 205         #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
 206         l.append('file=%s,snapshot=on'%self.image)
 207         #l.extend(['-vnc','127.0.0.1:1'])
 208         l.append('-nographic')
 209         print " ".join(l)
 210         self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
 211         time.sleep(10)
 212         signal.alarm(2400)
 213         signal.signal(signal.SIGALRM, self.timeout)
 214         try:
 215             self.run()
 216         finally:
 217             signal.signal(signal.SIGALRM, signal.SIG_DFL)
 218             os.kill(self.pid,15)
 219             time.sleep(10)
 220
 221     def ssh(self,cmd):
 222         l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
 223         system(l)
 224
 225     def rsync(self,args,options='--delete --exclude .bzrignore'):
 226         cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
 227         system(cmd)
 228
 229     def run(self):
 230         pass
 231
 232 class KVMWinBuildExe(KVM):
 233     def run(self):
 234         with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
 235             f.write("VERSION=%s\n" % self.o.version_full)
 236         with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
 237             f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
 238
 239         self.ssh("mkdir -p build")
 240         self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
 241         self.ssh("cd build/server/setup/win32;time make allinone;")
 242         self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
 243         print "KVMWinBuildExe.run(): done"
 244
 245 class KVMWinTestExe(KVM):
 246     def run(self):
 247         # Cannot use o.version_full when the version is not correctly parsed
 248         # (for instance, containing *rc* or *dev*)
 249         setuppath = glob("%s/openerp-server-setup-*.exe" % self.o.build_dir)[0]
 250         setupfile = setuppath.split('/')[-1]
 251         setupversion = setupfile.split('openerp-server-setup-')[1].split('.exe')[0]
 252
 253         self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
 254         self.ssh("TEMP=/tmp ./%s /S" % setupfile)
 255         self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Odoo %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
 256         self.ssh('/cygdrive/c/"Program Files"/"Odoo %s"/server/openerp-server.exe -d mycompany -i base --stop-after-init' % setupversion)
 257         self.ssh('net start odoo-server-8.0')
 258         _rpc_count_modules(port=18069)
 259
 260 #----------------------------------------------------------
 261 # Stage: building
 262 #----------------------------------------------------------
 263 def _prepare_build_dir(o):
 264     cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
 265     system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
 266     for i in glob(join(o.build_dir, 'addons/*')):
 267         shutil.move(i, join(o.build_dir, 'openerp/addons'))
 268
 269 def build_tgz(o):
 270     system(['python2', 'setup.py', '--quiet', 'sdist'], o.build_dir)
 271     system(['cp', glob('%s/dist/openerp-*.tar.gz' % o.build_dir)[0], '%s/odoo.tar.gz' % o.build_dir])
 272
 273 def build_deb(o):
 274     system(['dpkg-buildpackage', '-rfakeroot'], o.build_dir)
 275     system(['mv', glob('%s/../odoo_*.deb' % o.build_dir)[0], '%s' % o.build_dir])
 276     system(['mv', glob('%s/../odoo_*.dsc' % o.build_dir)[0], '%s' % o.build_dir])
 277     system(['mv', glob('%s/../odoo_*_amd64.changes' % o.build_dir)[0], '%s' % o.build_dir])
 278     system(['mv', glob('%s/../odoo_*.tar.gz' % o.build_dir)[0], '%s' % o.build_dir])
 279
 280 def build_rpm(o):
 281     system(['python2', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
 282     system(['cp', glob('%s/dist/openerp-*.noarch.rpm' % o.build_dir)[0], '%s/odoo.noarch.rpm' % o.build_dir])
 283     system(['cp', glob('%s/dist/openerp-*.src.rpm' % o.build_dir)[0], '%s/odoo.src.rpm' % o.build_dir])
 284
 285 def build_exe(o):
 286     KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 287     system(['cp', glob('%s/openerp*.exe' % o.build_dir)[0], '%s/odoo.exe' % o.build_dir])
 288
 289 #----------------------------------------------------------
 290 # Stage: testing
 291 #----------------------------------------------------------
 292 def test_tgz(o):
 293     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 294         wheezy.release = 'odoo.tar.gz'
 295         wheezy.system('apt-get update -qq && apt-get upgrade -qq -y')
 296         wheezy.system("apt-get install postgresql python-dev postgresql-server-dev-all python-pip build-essential libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libssl-dev libjpeg-dev -y")
 297         wheezy.system("service postgresql start")
 298         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 299         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 300         wheezy.system('pip install -r /opt/release/requirements.txt')
 301         wheezy.system('/usr/local/bin/pip install /opt/release/%s' % wheezy.release)
 302         wheezy.system("useradd --system --no-create-home odoo")
 303         wheezy.system('su postgres -s /bin/bash -c "createuser -s odoo"')
 304         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 305         wheezy.system('mkdir /var/lib/odoo')
 306         wheezy.system('chown odoo:odoo /var/lib/odoo')
 307         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany -i base --stop-after-init"')
 308         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany &"')
 309
 310 def test_deb(o):
 311     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 312         wheezy.release = '*.deb'
 313         wheezy.system('/usr/bin/apt-get update -qq && /usr/bin/apt-get upgrade -qq -y')
 314         wheezy.system("apt-get install postgresql -y")
 315         wheezy.system("service postgresql start")
 316         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 317         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 318         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 319         wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
 320         wheezy.system('/usr/bin/apt-get install -f -y')
 321         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 322         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany &"')
 323
 324 def test_rpm(o):
 325     with docker('centos:centos7', o.build_dir, o.pub) as centos7:
 326         centos7.release = 'odoo.noarch.rpm'
 327         # Dependencies
 328         centos7.system('yum install -d 0 -e 0 epel-release -y')
 329         centos7.system('yum update -d 0 -e 0 -y')
 330         # Manual install/start of postgres
 331         centos7.system('yum install -d 0 -e 0 postgresql postgresql-server postgresql-libs postgresql-contrib postgresql-devel -y')
 332         centos7.system('mkdir -p /var/lib/postgres/data')
 333         centos7.system('chown -R postgres:postgres /var/lib/postgres/data')
 334         centos7.system('chmod 0700 /var/lib/postgres/data')
 335         centos7.system('su postgres -c "initdb -D /var/lib/postgres/data -E UTF-8"')
 336         centos7.system('cp /usr/share/pgsql/postgresql.conf.sample /var/lib/postgres/data/postgresql.conf')
 337         centos7.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
 338         centos7.system('sleep 5')
 339         centos7.system('su postgres -c "createdb mycompany"')
 340         # Odoo install
 341         centos7.system('yum install -d 0 -e 0 /opt/release/%s -y' % centos7.release)
 342         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 343         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany &"')
 344
 345 def test_exe(o):
 346     KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 347
 348 #---------------------------------------------------------
 349 # Generates Packages, Sources and Release files of debian package
 350 #---------------------------------------------------------
 351 def gen_deb_package(o, published_files):
 352     # Executes command to produce file_name in path, and moves it to o.pub/deb
 353     def _gen_file(o, (command, file_name), path):
 354         cur_tmp_file_path = os.path.join(path, file_name)
 355         with open(cur_tmp_file_path, 'w') as out:
 356             subprocess.call(command, stdout=out, cwd=path)
 357         system(['cp', cur_tmp_file_path, os.path.join(o.pub, 'deb', file_name)])
 358
 359     # Copy files to a temp directory (required because the working directory must contain only the
 360     # files of the last release)
 361     temp_path = tempfile.mkdtemp(suffix='debPackages')
 362     for pub_file_path in published_files:
 363         system(['cp', pub_file_path, temp_path])
 364
 365     commands = [
 366         (['dpkg-scanpackages', '.'], "Packages"),  # Generate Packages file
 367         (['dpkg-scansources', '.'], "Sources"),  # Generate Sources file
 368         (['apt-ftparchive', 'release', '.'], "Release")  # Generate Release file
 369     ]
 370     # Generate files
 371     for command in commands:
 372         _gen_file(o, command, temp_path)
 373     # Remove temp directory
 374     shutil.rmtree(temp_path)
 375
 376     # Generate Release.gpg (= signed Release)
 377     # Options -abs: -a (Create ASCII armored output), -b (Make a detach signature), -s (Make a signature)
 378     subprocess.call(['gpg', '--yes', '-abs', '-o', 'Release.gpg', 'Release'], cwd=os.path.join(o.pub, 'deb'))
 379
 380 #---------------------------------------------------------
 381 # Generates an RPM repo
 382 #---------------------------------------------------------
 383 def gen_rpm_repo(o, file_name):
 384     # Sign the RPM
 385     subprocess.call(['rpm', '--resign', file_name], cwd=os.path.join(o.pub, 'rpm'))
 386
 387     # Removes the old repodata
 388     subprocess.call(['rm', '-rf', os.path.join(o.pub, 'rpm', 'repodata')])
 389
 390     # Copy files to a temp directory (required because the working directory must contain only the
 391     # files of the last release)
 392     temp_path = tempfile.mkdtemp(suffix='rpmPackages')
 393     subprocess.call(['cp', file_name, temp_path])
 394
 395     subprocess.call(['createrepo', temp_path])  # creates a repodata folder in temp_path
 396     subprocess.call(['cp', '-r', os.path.join(temp_path, "repodata"), os.path.join(o.pub, 'rpm')])
 397
 398     # Remove temp directory
 399     shutil.rmtree(temp_path)
 400
 401 #----------------------------------------------------------
 402 # Options and Main
 403 #----------------------------------------------------------
 404 def options():
 405     op = optparse.OptionParser()
 406     root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 407     build_dir = "%s-%s" % (root, timestamp)
 408
 409     op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
 410     op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
 411     op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
 412     op.add_option("-v", "--version", default='8.0', help="version (%default)")
 413
 414     op.add_option("", "--no-debian", action="store_true", help="don't build the debian package")
 415     op.add_option("", "--no-rpm", action="store_true", help="don't build the rpm package")
 416     op.add_option("", "--no-tarball", action="store_true", help="don't build the tarball")
 417     op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")
 418
 419     # Windows VM
 420     op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/winxp27/winxp27.vdi', help="%default")
 421     op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/winxp27/id_rsa', help="%default")
 422     op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
 423     op.add_option("", "--vm-winxp-python-version", default='2.7', help="Windows Python version installed in the VM (default: %default)")
 424
 425     (o, args) = op.parse_args()
 426     # derive other options
 427     o.odoo_dir = root
 428     o.pkg = join(o.build_dir, 'pkg')
 429     o.version_full = '%s-%s' % (o.version, timestamp)
 430     o.work = join(o.build_dir, 'openerp-%s' % o.version_full)
 431     o.work_addons = join(o.work, 'openerp', 'addons')
 432
 433     return o
 434
 435 def main():
 436     o = options()
 437     _prepare_build_dir(o)
 438     try:
 439         if not o.no_tarball:
 440             build_tgz(o)
 441             try:
 442                 if not o.no_testing:
 443                     test_tgz(o)
 444                 published_files = publish(o, 'tarball', ['odoo.tar.gz'])
 445             except Exception, e:
 446                 print("Won't publish the tgz release.\n Exception: %s" % str(e))
 447         if not o.no_debian:
 448             build_deb(o)
 449             try:
 450                 if not o.no_testing:
 451                     test_deb(o)
 452
 453                 to_publish = []
 454                 to_publish.append(glob("%s/odoo_*.deb" % o.build_dir)[0])
 455                 to_publish.append(glob("%s/odoo_*.dsc" % o.build_dir)[0])
 456                 to_publish.append(glob("%s/odoo_*.changes" % o.build_dir)[0])
 457                 to_publish.append(glob("%s/odoo_*.tar.gz" % o.build_dir)[0])
 458                 published_files = publish(o, 'debian', to_publish)
 459                 gen_deb_package(o, published_files)
 460             except Exception, e:
 461                 print("Won't publish the deb release.\n Exception: %s" % str(e))
 462         if not o.no_rpm:
 463             build_rpm(o)
 464             try:
 465                 if not o.no_testing:
 466                     test_rpm(o)
 467                 published_files = publish(o, 'redhat', ['odoo.noarch.rpm'])
 468                 gen_rpm_repo(o, published_files[0])
 469             except Exception, e:
 470                 print("Won't publish the rpm release.\n Exception: %s" % str(e))
 471         if not o.no_windows:
 472             build_exe(o)
 473             try:
 474                 if not o.no_testing:
 475                     test_exe(o)
 476                 published_files = publish(o, 'windows', ['odoo.exe'])
 477             except Exception, e:
 478                 print("Won't publish the exe release.\n Exception: %s" % str(e))
 479     except:
 480         pass
 481     finally:
 482         shutil.rmtree(o.build_dir)
 483         print('Build dir %s removed' % o.build_dir)
 484
 485         if not o.no_testing:
 486             system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
 487             print('Remaining dockers removed')
 488
 489
 490 if __name__ == '__main__':
 491     main()