setup/package.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 ##############################################################################
   4 #
   5 #    OpenERP, Open Source Management Solution
   6 #    Copyright (C) 2004-Today OpenERP SA (<http://www.openerp.com>).
   7 #
   8 #    This program is free software: you can redistribute it and/or modify
   9 #    it under the terms of the GNU Affero General Public License as
  10 #    published by the Free Software Foundation, either version 3 of the
  11 #    License, or (at your option) any later version.
  12 #
  13 #    This program is distributed in the hope that it will be useful,
  14 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #    GNU Affero General Public License for more details.
  17 #
  18 #    You should have received a copy of the GNU Affero General Public License
  19 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20 #
  21 ##############################################################################
  22
  23 import optparse
  24 import os
  25 import pexpect
  26 import shutil
  27 import signal
  28 import subprocess
  29 import tempfile
  30 import time
  31 import xmlrpclib
  32 from contextlib import contextmanager
  33 from glob import glob
  34 from os.path import abspath, dirname, join
  35 from tempfile import NamedTemporaryFile
  36
  37
  38 #----------------------------------------------------------
  39 # Utils
  40 #----------------------------------------------------------
  41 execfile(join(dirname(__file__), '..', 'openerp', 'release.py'))
  42 timestamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
  43 PUBLISH_DIRS = {
  44     'tar.gz': 'src',
  45     'exe': 'exe',
  46     'deb': 'deb',
  47     'dsc': 'deb',
  48     'changes': 'deb',
  49     'deb.tar.gz': ['deb', 'tar.gz'],
  50     'noarch.rpm': 'rpm',
  51     'src.rpm': 'rpm',
  52 }
  53
  54 def mkdir(d):
  55     if not os.path.isdir(d):
  56         os.makedirs(d)
  57
  58 def system(l, chdir=None):
  59     print l
  60     if chdir:
  61         cwd = os.getcwd()
  62         os.chdir(chdir)
  63     if isinstance(l, list):
  64         rc = os.spawnvp(os.P_WAIT, l[0], l)
  65     elif isinstance(l, str):
  66         tmp = ['sh', '-c', l]
  67         rc = os.spawnvp(os.P_WAIT, tmp[0], tmp)
  68     if chdir:
  69         os.chdir(cwd)
  70     return rc
  71
  72 def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
  73     time.sleep(5)
  74     modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  75         dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
  76     )
  77     if modules and len(modules) > 1:
  78         time.sleep(1)
  79         toinstallmodules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  80             dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
  81         )
  82         if toinstallmodules:
  83             print("Package test: FAILED. Not able to install dependencies of base.")
  84             raise Exception("Installation of package failed")
  85         else:
  86             print("Package test: successfuly installed %s modules" % len(modules))
  87     else:
  88         print("Package test: FAILED. Not able to install base.")
  89         raise Exception("Installation of package failed")
  90
  91 def publish(o, releases):
  92     def _publish(o, release):
  93         extension = ''.join(release.split('.', 1)[1])
  94         release_extension = PUBLISH_DIRS[extension][1] if isinstance(PUBLISH_DIRS[extension], list) else extension
  95         release_dir = PUBLISH_DIRS[extension][0] if isinstance(PUBLISH_DIRS[extension], list) else PUBLISH_DIRS[extension]
  96
  97         release_filename = 'odoo_%s-%s.%s' % (version, timestamp, release_extension)
  98         release_path = join(o.pub, release_dir, release_filename)
  99
 100         system('mkdir -p %s' % join(o.pub, release_dir))
 101         shutil.move(join(o.build_dir, release), release_path)
 102
 103         if release_extension == 'deb':
 104             temp_path = tempfile.mkdtemp(suffix='debPackages')
 105             system(['cp', release_path, temp_path])
 106             subprocess.Popen('dpkg-scanpackages . /dev/null | gzip -9c > %s' % join(o.pub, 'deb', 'Packages.gz'), shell=True, cwd=temp_path)
 107             shutil.rmtree(temp_path)
 108
 109         # Latest/symlink handler
 110         release_abspath = abspath(release_path)
 111         latest_abspath = release_abspath.replace(timestamp, 'latest')
 112
 113         if os.path.islink(latest_abspath):
 114             os.unlink(latest_abspath)
 115
 116         os.symlink(release_abspath, latest_abspath)
 117
 118     if isinstance(releases, basestring):
 119         _publish(o, releases)
 120     elif isinstance(releases, list):
 121         for release in releases:
 122             _publish(o, release)
 123
 124 class OdooDocker(object):
 125     def __init__(self):
 126         self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
 127         self.port = 8069  # TODO sle: reliable way to get a free port?
 128         self.prompt_re = '(\r\nroot@|bash-).*# '
 129         self.timeout = 600
 130
 131     def system(self, command):
 132         self.docker.sendline(command)
 133         self.docker.expect(self.prompt_re)
 134
 135     def start(self, docker_image, build_dir, pub_dir):
 136         self.build_dir = build_dir
 137         self.pub_dir = pub_dir
 138
 139         self.docker = pexpect.spawn(
 140             'docker run -v %s:/opt/release -p 127.0.0.1:%s:8069'
 141             ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
 142             timeout=self.timeout
 143         )
 144         time.sleep(2)  # let the bash start
 145         self.docker.logfile_read = self.log_file
 146         self.id = subprocess.check_output('docker ps -l -q', shell=True)
 147
 148     def end(self):
 149         try:
 150             _rpc_count_modules(port=str(self.port))
 151         except Exception, e:
 152             print('Exception during docker execution: %s:' % str(e))
 153             print('Error during docker execution: printing the bash output:')
 154             with open(self.log_file.name) as f:
 155                 print '\n'.join(f.readlines())
 156             raise
 157         finally:
 158             self.docker.close()
 159             system('docker rm -f %s' % self.id)
 160             self.log_file.close()
 161             os.remove(self.log_file.name)
 162
 163 @contextmanager
 164 def docker(docker_image, build_dir, pub_dir):
 165     _docker = OdooDocker()
 166     try:
 167         _docker.start(docker_image, build_dir, pub_dir)
 168         try:
 169             yield _docker
 170         except Exception, e:
 171             raise
 172     finally:
 173         _docker.end()
 174
 175 class KVM(object):
 176     def __init__(self, o, image, ssh_key='', login='openerp'):
 177         self.o = o
 178         self.image = image
 179         self.ssh_key = ssh_key
 180         self.login = login
 181
 182     def timeout(self,signum,frame):
 183         print "vm timeout kill",self.pid
 184         os.kill(self.pid,15)
 185
 186     def start(self):
 187         l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -drive".split(" ")
 188         #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
 189         l.append('file=%s,snapshot=on'%self.image)
 190         #l.extend(['-vnc','127.0.0.1:1'])
 191         l.append('-nographic')
 192         print " ".join(l)
 193         self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
 194         time.sleep(10)
 195         signal.alarm(2400)
 196         signal.signal(signal.SIGALRM, self.timeout)
 197         try:
 198             self.run()
 199         finally:
 200             signal.signal(signal.SIGALRM, signal.SIG_DFL)
 201             os.kill(self.pid,15)
 202             time.sleep(10)
 203
 204     def ssh(self,cmd):
 205         l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
 206         system(l)
 207
 208     def rsync(self,args,options='--delete --exclude .bzrignore'):
 209         cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
 210         system(cmd)
 211
 212     def run(self):
 213         pass
 214
 215 class KVMWinBuildExe(KVM):
 216     def run(self):
 217         with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
 218             f.write("VERSION=%s\n" % self.o.version_full)
 219         with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
 220             f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
 221
 222         self.ssh("mkdir -p build")
 223         self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
 224         self.ssh("cd build/server/setup/win32;time make allinone;")
 225         self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
 226         print "KVMWinBuildExe.run(): done"
 227
 228 class KVMWinTestExe(KVM):
 229     def run(self):
 230         # Cannot use o.version_full when the version is not correctly parsed
 231         # (for instance, containing *rc* or *dev*)
 232         setuppath = glob("%s/openerp-server-setup-*.exe" % self.o.build_dir)[0]
 233         setupfile = setuppath.split('/')[-1]
 234         setupversion = setupfile.split('openerp-server-setup-')[1].split('.exe')[0]
 235
 236         self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
 237         self.ssh("TEMP=/tmp ./%s /S" % setupfile)
 238         self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Odoo %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
 239         self.ssh('/cygdrive/c/"Program Files"/"Odoo %s"/server/openerp-server.exe -d mycompany -i base --stop-after-init' % setupversion)
 240         self.ssh('net start odoo-server-8.0')
 241         _rpc_count_modules(port=18069)
 242
 243 #----------------------------------------------------------
 244 # Stage: building
 245 #----------------------------------------------------------
 246 def _prepare_build_dir(o):
 247     cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
 248     system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
 249     for i in glob(join(o.build_dir, 'addons/*')):
 250         shutil.move(i, join(o.build_dir, 'openerp/addons'))
 251
 252 def build_tgz(o):
 253     system(['python2', 'setup.py', '--quiet', 'sdist'], o.build_dir)
 254     system(['cp', glob('%s/dist/openerp-*.tar.gz' % o.build_dir)[0], '%s/odoo.tar.gz' % o.build_dir])
 255
 256 def build_deb(o):
 257     system(['dpkg-buildpackage', '-rfakeroot', '-uc', '-us'], o.build_dir)
 258     system(['cp', glob('%s/../odoo_*.deb' % o.build_dir)[0], '%s/odoo.deb' % o.build_dir])
 259     system(['cp', glob('%s/../odoo_*.dsc' % o.build_dir)[0], '%s/odoo.dsc' % o.build_dir])
 260     system(['cp', glob('%s/../odoo_*_amd64.changes' % o.build_dir)[0], '%s/odoo_amd64.changes' % o.build_dir])
 261     system(['cp', glob('%s/../odoo_*.tar.gz' % o.build_dir)[0], '%s/odoo.deb.tar.gz' % o.build_dir])
 262
 263 def build_rpm(o):
 264     system(['python2', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
 265     system(['cp', glob('%s/dist/openerp-*.noarch.rpm' % o.build_dir)[0], '%s/odoo.noarch.rpm' % o.build_dir])
 266     system(['cp', glob('%s/dist/openerp-*.src.rpm' % o.build_dir)[0], '%s/odoo.src.rpm' % o.build_dir])
 267
 268 def build_exe(o):
 269     KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 270     system(['cp', glob('%s/openerp*.exe' % o.build_dir)[0], '%s/odoo.exe' % o.build_dir])
 271
 272 #----------------------------------------------------------
 273 # Stage: testing
 274 #----------------------------------------------------------
 275 def test_tgz(o):
 276     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 277         wheezy.release = 'odoo.tar.gz'
 278         wheezy.system('apt-get update -qq && apt-get upgrade -qq -y')
 279         wheezy.system("apt-get install postgresql python-dev postgresql-server-dev-all python-pip build-essential libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libssl-dev libjpeg-dev -y")
 280         wheezy.system("service postgresql start")
 281         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 282         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 283         wheezy.system('pip install -r /opt/release/requirements.txt')
 284         wheezy.system('/usr/local/bin/pip install /opt/release/%s' % wheezy.release)
 285         wheezy.system("useradd --system --no-create-home odoo")
 286         wheezy.system('su postgres -s /bin/bash -c "createuser -s odoo"')
 287         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 288         wheezy.system('mkdir /var/lib/odoo')
 289         wheezy.system('chown odoo:odoo /var/lib/odoo')
 290         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany -i base --stop-after-init"')
 291         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany &"')
 292
 293 def test_deb(o):
 294     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 295         wheezy.release = 'odoo.deb'
 296         wheezy.system('/usr/bin/apt-get update -qq && /usr/bin/apt-get upgrade -qq -y')
 297         wheezy.system("apt-get install postgresql -y")
 298         wheezy.system("service postgresql start")
 299         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 300         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 301         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 302         wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
 303         wheezy.system('/usr/bin/apt-get install -f -y')
 304         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 305         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany &"')
 306
 307 def test_rpm(o):
 308     with docker('centos:centos7', o.build_dir, o.pub) as centos7:
 309         centos7.release = 'odoo.noarch.rpm'
 310         centos7.system('rpm -Uvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-1.noarch.rpm')
 311         centos7.system('yum update -y && yum upgrade -y')
 312         centos7.system('yum install python-pip gcc python-devel -y')
 313         centos7.system('pip install pydot pyPdf vatnumber xlwt http://download.gna.org/pychart/PyChart-1.39.tar.gz')
 314         centos7.system('yum install postgresql postgresql-server postgresql-libs postgresql-contrib postgresql-devel -y')
 315         centos7.system('mkdir -p /var/lib/postgres/data')
 316         centos7.system('chown -R postgres:postgres /var/lib/postgres/data')
 317         centos7.system('chmod 0700 /var/lib/postgres/data')
 318         centos7.system('su postgres -c "initdb -D /var/lib/postgres/data -E UTF-8"')
 319         centos7.system('cp /usr/share/pgsql/postgresql.conf.sample /var/lib/postgres/data/postgresql.conf')
 320         centos7.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
 321         centos7.system('su postgres -c "createdb mycompany"')
 322         centos7.system('export PYTHONPATH=${PYTHONPATH}:/usr/local/lib/python2.7/dist-packages')
 323         centos7.system('su postgres -c "createdb mycompany"')
 324         centos7.system('yum install /opt/release/%s -y' % centos7.release)
 325         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 326         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany &"')
 327
 328 def test_exe(o):
 329     KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 330
 331 #----------------------------------------------------------
 332 # Options and Main
 333 #----------------------------------------------------------
 334 def options():
 335     op = optparse.OptionParser()
 336     root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 337     build_dir = "%s-%s" % (root, timestamp)
 338
 339     op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
 340     op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
 341     op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
 342     op.add_option("-v", "--version", default='8.0', help="version (%default)")
 343
 344     op.add_option("", "--no-debian", action="store_true", help="don't build the debian package")
 345     op.add_option("", "--no-rpm", action="store_true", help="don't build the rpm package")
 346     op.add_option("", "--no-tarball", action="store_true", help="don't build the tarball")
 347     op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")
 348
 349     # Windows VM
 350     op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/winxp27/winxp27.vdi', help="%default")
 351     op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/winxp27/id_rsa', help="%default")
 352     op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
 353     op.add_option("", "--vm-winxp-python-version", default='2.7', help="Windows Python version installed in the VM (default: %default)")
 354
 355     (o, args) = op.parse_args()
 356     # derive other options
 357     o.odoo_dir = root
 358     o.pkg = join(o.build_dir, 'pkg')
 359     o.version_full = '%s-%s' % (o.version, timestamp)
 360     o.work = join(o.build_dir, 'openerp-%s' % o.version_full)
 361     o.work_addons = join(o.work, 'openerp', 'addons')
 362     return o
 363
 364 def main():
 365     o = options()
 366     _prepare_build_dir(o)
 367     try:
 368         if not o.no_tarball:
 369             build_tgz(o)
 370             if not o.no_testing:
 371                 try:
 372                     test_tgz(o)
 373                     publish(o, 'odoo.tar.gz')
 374                 except Exception, e:
 375                     print("Won't publish the tgz release.\n Exception: %s" % str(e))
 376         if not o.no_debian:
 377             build_deb(o)
 378             if not o.no_testing:
 379                 try:
 380                     test_deb(o)
 381                     publish(o, ['odoo.deb', 'odoo.dsc', 'odoo_amd64.changes', 'odoo.deb.tar.gz'])
 382                 except Exception, e:
 383                     print("Won't publish the deb release.\n Exception: %s" % str(e))
 384         if not o.no_rpm:
 385             build_rpm(o)
 386             if not o.no_testing:
 387                 try:
 388                     test_rpm(o)
 389                     publish(o, ['odoo.noarch.rpm', 'odoo.src.rpm'])
 390                 except Exception, e:
 391                     print("Won't publish the rpm release.\n Exception: %s" % str(e))
 392         if not o.no_windows:
 393             build_exe(o)
 394             if not o.no_testing:
 395                 try:
 396                     test_exe(o)
 397                     publish(o, 'odoo.exe')
 398                 except Exception, e:
 399                     print("Won't publish the exe release.\n Exception: %s" % str(e))
 400     except:
 401         pass
 402     finally:
 403         for leftover in glob('%s/../odoo_*' % o.build_dir):
 404             os.remove(leftover)
 405
 406         shutil.rmtree(o.build_dir)
 407         print('Build dir %s removed' % o.build_dir)
 408
 409         if not o.no_testing:
 410             system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
 411             print('Remaining dockers removed')
 412
 413
 414 if __name__ == '__main__':
 415     main()