setup/package.py

   1 #!/usr/bin/env python2
   2 # -*- coding: utf-8 -*-
   3 ##############################################################################
   4 #
   5 #    OpenERP, Open Source Management Solution
   6 #    Copyright (C) 2004-Today OpenERP SA (<http://www.openerp.com>).
   7 #
   8 #    This program is free software: you can redistribute it and/or modify
   9 #    it under the terms of the GNU Affero General Public License as
  10 #    published by the Free Software Foundation, either version 3 of the
  11 #    License, or (at your option) any later version.
  12 #
  13 #    This program is distributed in the hope that it will be useful,
  14 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #    GNU Affero General Public License for more details.
  17 #
  18 #    You should have received a copy of the GNU Affero General Public License
  19 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  20 #
  21 ##############################################################################
  22
  23 import optparse
  24 import os
  25 import pexpect
  26 import shutil
  27 import signal
  28 import subprocess
  29 import tempfile
  30 import time
  31 import xmlrpclib
  32 from contextlib import contextmanager
  33 from glob import glob
  34 from os.path import abspath, dirname, join
  35 from tempfile import NamedTemporaryFile
  36
  37
  38 #----------------------------------------------------------
  39 # Utils
  40 #----------------------------------------------------------
  41 execfile(join(dirname(__file__), '..', 'openerp', 'release.py'))
  42 timestamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
  43 PUBLISH_DIRS = {
  44     'tar.gz': 'src',
  45     'exe': 'exe',
  46     'deb': 'deb',
  47     'dsc': 'deb',
  48     'changes': 'deb',
  49     'deb.tar.gz': ['deb', 'tar.gz'],
  50     'noarch.rpm': 'rpm',
  51     'src.rpm': 'rpm',
  52 }
  53
  54 def mkdir(d):
  55     if not os.path.isdir(d):
  56         os.makedirs(d)
  57
  58 def system(l, chdir=None):
  59     print l
  60     if chdir:
  61         cwd = os.getcwd()
  62         os.chdir(chdir)
  63     if isinstance(l, list):
  64         rc = os.spawnvp(os.P_WAIT, l[0], l)
  65     elif isinstance(l, str):
  66         tmp = ['sh', '-c', l]
  67         rc = os.spawnvp(os.P_WAIT, tmp[0], tmp)
  68     if chdir:
  69         os.chdir(cwd)
  70     return rc
  71
  72 def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
  73     time.sleep(5)
  74     modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  75         dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
  76     )
  77     if modules and len(modules) > 1:
  78         time.sleep(1)
  79         toinstallmodules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
  80             dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
  81         )
  82         if toinstallmodules:
  83             print("Package test: FAILED. Not able to install dependencies of base.")
  84             raise Exception("Installation of package failed")
  85         else:
  86             print("Package test: successfuly installed %s modules" % len(modules))
  87     else:
  88         print("Package test: FAILED. Not able to install base.")
  89         raise Exception("Installation of package failed")
  90
  91 def publish(o, releases):
  92     def _publish(o, release):
  93         extension = ''.join(release.split('.', 1)[1])
  94         release_extension = PUBLISH_DIRS[extension][1] if isinstance(PUBLISH_DIRS[extension], list) else extension
  95         release_dir = PUBLISH_DIRS[extension][0] if isinstance(PUBLISH_DIRS[extension], list) else PUBLISH_DIRS[extension]
  96
  97         release_filename = 'odoo_%s-%s.%s' % (version, timestamp, release_extension)
  98         release_path = join(o.pub, release_dir, release_filename)
  99
 100         system('mkdir -p %s' % join(o.pub, release_dir))
 101         shutil.move(join(o.build_dir, release), release_path)
 102
 103         if release_extension == 'deb':
 104             temp_path = tempfile.mkdtemp(suffix='debPackages')
 105             system(['cp', release_path, temp_path])
 106             with open(os.path.join(o.pub, 'deb', 'Packages'), 'w') as out:
 107                 subprocess.call(['dpkg-scanpackages', '.'], stdout=out, cwd=temp_path)
 108             shutil.rmtree(temp_path)
 109
 110         # Latest/symlink handler
 111         release_abspath = abspath(release_path)
 112         latest_abspath = release_abspath.replace(timestamp, 'latest')
 113
 114         if os.path.islink(latest_abspath):
 115             os.unlink(latest_abspath)
 116
 117         os.symlink(release_abspath, latest_abspath)
 118
 119     if isinstance(releases, basestring):
 120         _publish(o, releases)
 121     elif isinstance(releases, list):
 122         for release in releases:
 123             _publish(o, release)
 124
 125 class OdooDocker(object):
 126     def __init__(self):
 127         self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
 128         self.port = 8069  # TODO sle: reliable way to get a free port?
 129         self.prompt_re = '(\r\nroot@|bash-).*# '
 130         self.timeout = 600
 131
 132     def system(self, command):
 133         self.docker.sendline(command)
 134         self.docker.expect(self.prompt_re)
 135
 136     def start(self, docker_image, build_dir, pub_dir):
 137         self.build_dir = build_dir
 138         self.pub_dir = pub_dir
 139
 140         self.docker = pexpect.spawn(
 141             'docker run -v %s:/opt/release -p 127.0.0.1:%s:8069'
 142             ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
 143             timeout=self.timeout
 144         )
 145         time.sleep(2)  # let the bash start
 146         self.docker.logfile_read = self.log_file
 147         self.id = subprocess.check_output('docker ps -l -q', shell=True)
 148
 149     def end(self):
 150         try:
 151             _rpc_count_modules(port=str(self.port))
 152         except Exception, e:
 153             print('Exception during docker execution: %s:' % str(e))
 154             print('Error during docker execution: printing the bash output:')
 155             with open(self.log_file.name) as f:
 156                 print '\n'.join(f.readlines())
 157             raise
 158         finally:
 159             self.docker.close()
 160             system('docker rm -f %s' % self.id)
 161             self.log_file.close()
 162             os.remove(self.log_file.name)
 163
 164 @contextmanager
 165 def docker(docker_image, build_dir, pub_dir):
 166     _docker = OdooDocker()
 167     try:
 168         _docker.start(docker_image, build_dir, pub_dir)
 169         try:
 170             yield _docker
 171         except Exception, e:
 172             raise
 173     finally:
 174         _docker.end()
 175
 176 class KVM(object):
 177     def __init__(self, o, image, ssh_key='', login='openerp'):
 178         self.o = o
 179         self.image = image
 180         self.ssh_key = ssh_key
 181         self.login = login
 182
 183     def timeout(self,signum,frame):
 184         print "vm timeout kill",self.pid
 185         os.kill(self.pid,15)
 186
 187     def start(self):
 188         l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -drive".split(" ")
 189         #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
 190         l.append('file=%s,snapshot=on'%self.image)
 191         #l.extend(['-vnc','127.0.0.1:1'])
 192         l.append('-nographic')
 193         print " ".join(l)
 194         self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
 195         time.sleep(10)
 196         signal.alarm(2400)
 197         signal.signal(signal.SIGALRM, self.timeout)
 198         try:
 199             self.run()
 200         finally:
 201             signal.signal(signal.SIGALRM, signal.SIG_DFL)
 202             os.kill(self.pid,15)
 203             time.sleep(10)
 204
 205     def ssh(self,cmd):
 206         l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
 207         system(l)
 208
 209     def rsync(self,args,options='--delete --exclude .bzrignore'):
 210         cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
 211         system(cmd)
 212
 213     def run(self):
 214         pass
 215
 216 class KVMWinBuildExe(KVM):
 217     def run(self):
 218         with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
 219             f.write("VERSION=%s\n" % self.o.version_full)
 220         with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
 221             f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
 222
 223         self.ssh("mkdir -p build")
 224         self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
 225         self.ssh("cd build/server/setup/win32;time make allinone;")
 226         self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
 227         print "KVMWinBuildExe.run(): done"
 228
 229 class KVMWinTestExe(KVM):
 230     def run(self):
 231         # Cannot use o.version_full when the version is not correctly parsed
 232         # (for instance, containing *rc* or *dev*)
 233         setuppath = glob("%s/openerp-server-setup-*.exe" % self.o.build_dir)[0]
 234         setupfile = setuppath.split('/')[-1]
 235         setupversion = setupfile.split('openerp-server-setup-')[1].split('.exe')[0]
 236
 237         self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
 238         self.ssh("TEMP=/tmp ./%s /S" % setupfile)
 239         self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Odoo %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
 240         self.ssh('/cygdrive/c/"Program Files"/"Odoo %s"/server/openerp-server.exe -d mycompany -i base --stop-after-init' % setupversion)
 241         self.ssh('net start odoo-server-8.0')
 242         _rpc_count_modules(port=18069)
 243
 244 #----------------------------------------------------------
 245 # Stage: building
 246 #----------------------------------------------------------
 247 def _prepare_build_dir(o):
 248     cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
 249     system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
 250     for i in glob(join(o.build_dir, 'addons/*')):
 251         shutil.move(i, join(o.build_dir, 'openerp/addons'))
 252
 253 def build_tgz(o):
 254     system(['python2', 'setup.py', '--quiet', 'sdist'], o.build_dir)
 255     system(['cp', glob('%s/dist/openerp-*.tar.gz' % o.build_dir)[0], '%s/odoo.tar.gz' % o.build_dir])
 256
 257 def build_deb(o):
 258     system(['dpkg-buildpackage', '-rfakeroot', '-uc', '-us'], o.build_dir)
 259     system(['cp', glob('%s/../odoo_*.deb' % o.build_dir)[0], '%s/odoo.deb' % o.build_dir])
 260     system(['cp', glob('%s/../odoo_*.dsc' % o.build_dir)[0], '%s/odoo.dsc' % o.build_dir])
 261     system(['cp', glob('%s/../odoo_*_amd64.changes' % o.build_dir)[0], '%s/odoo_amd64.changes' % o.build_dir])
 262     system(['cp', glob('%s/../odoo_*.tar.gz' % o.build_dir)[0], '%s/odoo.deb.tar.gz' % o.build_dir])
 263
 264 def build_rpm(o):
 265     system(['python2', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
 266     system(['cp', glob('%s/dist/openerp-*.noarch.rpm' % o.build_dir)[0], '%s/odoo.noarch.rpm' % o.build_dir])
 267     system(['cp', glob('%s/dist/openerp-*.src.rpm' % o.build_dir)[0], '%s/odoo.src.rpm' % o.build_dir])
 268
 269 def build_exe(o):
 270     KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 271     system(['cp', glob('%s/openerp*.exe' % o.build_dir)[0], '%s/odoo.exe' % o.build_dir])
 272
 273 #----------------------------------------------------------
 274 # Stage: testing
 275 #----------------------------------------------------------
 276 def test_tgz(o):
 277     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 278         wheezy.release = 'odoo.tar.gz'
 279         wheezy.system('apt-get update -qq && apt-get upgrade -qq -y')
 280         wheezy.system("apt-get install postgresql python-dev postgresql-server-dev-all python-pip build-essential libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libssl-dev libjpeg-dev -y")
 281         wheezy.system("service postgresql start")
 282         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 283         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 284         wheezy.system('pip install -r /opt/release/requirements.txt')
 285         wheezy.system('/usr/local/bin/pip install /opt/release/%s' % wheezy.release)
 286         wheezy.system("useradd --system --no-create-home odoo")
 287         wheezy.system('su postgres -s /bin/bash -c "createuser -s odoo"')
 288         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 289         wheezy.system('mkdir /var/lib/odoo')
 290         wheezy.system('chown odoo:odoo /var/lib/odoo')
 291         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany -i base --stop-after-init"')
 292         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany &"')
 293
 294 def test_deb(o):
 295     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
 296         wheezy.release = 'odoo.deb'
 297         wheezy.system('/usr/bin/apt-get update -qq && /usr/bin/apt-get upgrade -qq -y')
 298         wheezy.system("apt-get install postgresql -y")
 299         wheezy.system("service postgresql start")
 300         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
 301         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
 302         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
 303         wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
 304         wheezy.system('/usr/bin/apt-get install -f -y')
 305         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 306         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany &"')
 307
 308 def test_rpm(o):
 309     with docker('centos:centos7', o.build_dir, o.pub) as centos7:
 310         centos7.release = 'odoo.noarch.rpm'
 311         centos7.system('rpm -Uvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-1.noarch.rpm')
 312         centos7.system('yum update -y && yum upgrade -y')
 313         centos7.system('yum install python-pip gcc python-devel -y')
 314         centos7.system('pip install pydot pyPdf vatnumber xlwt http://download.gna.org/pychart/PyChart-1.39.tar.gz')
 315         centos7.system('yum install postgresql postgresql-server postgresql-libs postgresql-contrib postgresql-devel -y')
 316         centos7.system('mkdir -p /var/lib/postgres/data')
 317         centos7.system('chown -R postgres:postgres /var/lib/postgres/data')
 318         centos7.system('chmod 0700 /var/lib/postgres/data')
 319         centos7.system('su postgres -c "initdb -D /var/lib/postgres/data -E UTF-8"')
 320         centos7.system('cp /usr/share/pgsql/postgresql.conf.sample /var/lib/postgres/data/postgresql.conf')
 321         centos7.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
 322         centos7.system('su postgres -c "createdb mycompany"')
 323         centos7.system('export PYTHONPATH=${PYTHONPATH}:/usr/local/lib/python2.7/dist-packages')
 324         centos7.system('su postgres -c "createdb mycompany"')
 325         centos7.system('yum install /opt/release/%s -y' % centos7.release)
 326         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
 327         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany &"')
 328
 329 def test_exe(o):
 330     KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
 331
 332 #----------------------------------------------------------
 333 # Options and Main
 334 #----------------------------------------------------------
 335 def options():
 336     op = optparse.OptionParser()
 337     root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 338     build_dir = "%s-%s" % (root, timestamp)
 339
 340     op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
 341     op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
 342     op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
 343     op.add_option("-v", "--version", default='8.0', help="version (%default)")
 344
 345     op.add_option("", "--no-debian", action="store_true", help="don't build the debian package")
 346     op.add_option("", "--no-rpm", action="store_true", help="don't build the rpm package")
 347     op.add_option("", "--no-tarball", action="store_true", help="don't build the tarball")
 348     op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")
 349
 350     # Windows VM
 351     op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/winxp27/winxp27.vdi', help="%default")
 352     op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/winxp27/id_rsa', help="%default")
 353     op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
 354     op.add_option("", "--vm-winxp-python-version", default='2.7', help="Windows Python version installed in the VM (default: %default)")
 355
 356     (o, args) = op.parse_args()
 357     # derive other options
 358     o.odoo_dir = root
 359     o.pkg = join(o.build_dir, 'pkg')
 360     o.version_full = '%s-%s' % (o.version, timestamp)
 361     o.work = join(o.build_dir, 'openerp-%s' % o.version_full)
 362     o.work_addons = join(o.work, 'openerp', 'addons')
 363     return o
 364
 365 def main():
 366     o = options()
 367     _prepare_build_dir(o)
 368     try:
 369         if not o.no_tarball:
 370             build_tgz(o)
 371             if not o.no_testing:
 372                 try:
 373                     test_tgz(o)
 374                     publish(o, 'odoo.tar.gz')
 375                 except Exception, e:
 376                     print("Won't publish the tgz release.\n Exception: %s" % str(e))
 377         if not o.no_debian:
 378             build_deb(o)
 379             if not o.no_testing:
 380                 try:
 381                     test_deb(o)
 382                     publish(o, ['odoo.deb', 'odoo.dsc', 'odoo_amd64.changes', 'odoo.deb.tar.gz'])
 383                 except Exception, e:
 384                     print("Won't publish the deb release.\n Exception: %s" % str(e))
 385         if not o.no_rpm:
 386             build_rpm(o)
 387             if not o.no_testing:
 388                 try:
 389                     test_rpm(o)
 390                     publish(o, ['odoo.noarch.rpm', 'odoo.src.rpm'])
 391                 except Exception, e:
 392                     print("Won't publish the rpm release.\n Exception: %s" % str(e))
 393         if not o.no_windows:
 394             build_exe(o)
 395             if not o.no_testing:
 396                 try:
 397                     test_exe(o)
 398                     publish(o, 'odoo.exe')
 399                 except Exception, e:
 400                     print("Won't publish the exe release.\n Exception: %s" % str(e))
 401     except:
 402         pass
 403     finally:
 404         for leftover in glob('%s/../odoo_*' % o.build_dir):
 405             os.remove(leftover)
 406
 407         shutil.rmtree(o.build_dir)
 408         print('Build dir %s removed' % o.build_dir)
 409
 410         if not o.no_testing:
 411             system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
 412             print('Remaining dockers removed')
 413
 414
 415 if __name__ == '__main__':
 416     main()