[IMP] Rounding should be done on move immediately to default UoM and quants should...
[odoo/odoo.git] / setup / package.py
1 #!/usr/bin/env python2
2 # -*- coding: utf-8 -*-
3 ##############################################################################
4 #
5 #    OpenERP, Open Source Management Solution
6 #    Copyright (C) 2004-Today OpenERP SA (<http://www.openerp.com>).
7 #
8 #    This program is free software: you can redistribute it and/or modify
9 #    it under the terms of the GNU Affero General Public License as
10 #    published by the Free Software Foundation, either version 3 of the
11 #    License, or (at your option) any later version.
12 #
13 #    This program is distributed in the hope that it will be useful,
14 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
15 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 #    GNU Affero General Public License for more details.
17 #
18 #    You should have received a copy of the GNU Affero General Public License
19 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
20 #
21 ##############################################################################
22
23 import optparse
24 import os
25 import pexpect
26 import shutil
27 import signal
28 import subprocess
29 import tempfile
30 import time
31 import xmlrpclib
32 from contextlib import contextmanager
33 from glob import glob
34 from os.path import abspath, dirname, join
35 from sys import stdout
36 from tempfile import NamedTemporaryFile
37
38
39 #----------------------------------------------------------
40 # Utils
41 #----------------------------------------------------------
42 execfile(join(dirname(__file__), '..', 'openerp', 'release.py'))
43 version = version.split('-')[0]
44 GPGPASSPHRASE = os.getenv('GPGPASSPHRASE')
45 GPGID = os.getenv('GPGID')
46 timestamp = time.strftime("%Y%m%d", time.gmtime())
47 PUBLISH_DIRS = {
48     'debian': 'deb',
49     'redhat': 'rpm',
50     'tarball': 'src',
51     'windows': 'exe',
52 }
53 EXTENSIONS = [
54     '.tar.gz',
55     '.deb',
56     '.dsc',
57     '.changes',
58     '.noarch.rpm',
59     '.exe',
60 ]
61
62 def mkdir(d):
63     if not os.path.isdir(d):
64         os.makedirs(d)
65
66 def system(l, chdir=None):
67     print l
68     if chdir:
69         cwd = os.getcwd()
70         os.chdir(chdir)
71     if isinstance(l, list):
72         rc = os.spawnvp(os.P_WAIT, l[0], l)
73     elif isinstance(l, str):
74         tmp = ['sh', '-c', l]
75         rc = os.spawnvp(os.P_WAIT, tmp[0], tmp)
76     if chdir:
77         os.chdir(cwd)
78     return rc
79
80 def _rpc_count_modules(addr='http://127.0.0.1', port=8069, dbname='mycompany'):
81     time.sleep(5)
82     modules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
83         dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'installed')]
84     )
85     if modules and len(modules) > 1:
86         time.sleep(1)
87         toinstallmodules = xmlrpclib.ServerProxy('%s:%s/xmlrpc/object' % (addr, port)).execute(
88             dbname, 1, 'admin', 'ir.module.module', 'search', [('state', '=', 'to install')]
89         )
90         if toinstallmodules:
91             print("Package test: FAILED. Not able to install dependencies of base.")
92             raise Exception("Installation of package failed")
93         else:
94             print("Package test: successfuly installed %s modules" % len(modules))
95     else:
96         print("Package test: FAILED. Not able to install base.")
97         raise Exception("Installation of package failed")
98
99 def publish(o, type, releases):
100     def _publish(o, release):
101         arch = ''
102         filename = release.split(os.path.sep)[-1]
103
104         extension = None
105         for EXTENSION in EXTENSIONS:
106             if filename.endswith(EXTENSION):
107                 extension = EXTENSION
108                 filename = filename.replace(extension, '')
109                 break
110         if extension is None:
111             raise Exception("Extension of %s is not handled" % filename)
112
113         # keep _all or _amd64
114         if filename.count('_') > 1:
115             arch = '_' + filename.split('_')[-1]
116
117         release_dir = PUBLISH_DIRS[type]
118         release_filename = 'odoo_%s.%s%s%s' % (version, timestamp, arch, extension)
119         release_path = join(o.pub, release_dir, release_filename)
120
121         system('mkdir -p %s' % join(o.pub, release_dir))
122         shutil.move(join(o.build_dir, release), release_path)
123
124         # Latest/symlink handler
125         release_abspath = abspath(release_path)
126         latest_abspath = release_abspath.replace(timestamp, 'latest')
127
128         if os.path.islink(latest_abspath):
129             os.unlink(latest_abspath)
130
131         os.symlink(release_abspath, latest_abspath)
132
133         return release_path
134
135     published = []
136     if isinstance(releases, basestring):
137         published.append(_publish(o, releases))
138     elif isinstance(releases, list):
139         for release in releases:
140             published.append(_publish(o, release))
141     return published
142
143 class OdooDocker(object):
144     def __init__(self):
145         self.log_file = NamedTemporaryFile(mode='w+b', prefix="bash", suffix=".txt", delete=False)
146         self.port = 8069  # TODO sle: reliable way to get a free port?
147         self.prompt_re = '(\r\nroot@|bash-).*# '
148         self.timeout = 600
149
150     def system(self, command):
151         self.docker.sendline(command)
152         self.docker.expect(self.prompt_re)
153
154     def start(self, docker_image, build_dir, pub_dir):
155         self.build_dir = build_dir
156         self.pub_dir = pub_dir
157
158         self.docker = pexpect.spawn(
159             'docker run -v %s:/opt/release -p 127.0.0.1:%s:8069'
160             ' -t -i %s /bin/bash --noediting' % (self.build_dir, self.port, docker_image),
161             timeout=self.timeout
162         )
163         time.sleep(2)  # let the bash start
164         self.docker.logfile_read = self.log_file
165         self.id = subprocess.check_output('docker ps -l -q', shell=True)
166
167     def end(self):
168         try:
169             _rpc_count_modules(port=str(self.port))
170         except Exception, e:
171             print('Exception during docker execution: %s:' % str(e))
172             print('Error during docker execution: printing the bash output:')
173             with open(self.log_file.name) as f:
174                 print '\n'.join(f.readlines())
175             raise
176         finally:
177             self.docker.close()
178             system('docker rm -f %s' % self.id)
179             self.log_file.close()
180             os.remove(self.log_file.name)
181
182 @contextmanager
183 def docker(docker_image, build_dir, pub_dir):
184     _docker = OdooDocker()
185     try:
186         _docker.start(docker_image, build_dir, pub_dir)
187         try:
188             yield _docker
189         except Exception, e:
190             raise
191     finally:
192         _docker.end()
193
194 class KVM(object):
195     def __init__(self, o, image, ssh_key='', login='openerp'):
196         self.o = o
197         self.image = image
198         self.ssh_key = ssh_key
199         self.login = login
200
201     def timeout(self,signum,frame):
202         print "vm timeout kill",self.pid
203         os.kill(self.pid,15)
204
205     def start(self):
206         l="kvm -net nic,model=rtl8139 -net user,hostfwd=tcp:127.0.0.1:10022-:22,hostfwd=tcp:127.0.0.1:18069-:8069,hostfwd=tcp:127.0.0.1:15432-:5432 -drive".split(" ")
207         #l.append('file=%s,if=virtio,index=0,boot=on,snapshot=on'%self.image)
208         l.append('file=%s,snapshot=on'%self.image)
209         #l.extend(['-vnc','127.0.0.1:1'])
210         l.append('-nographic')
211         print " ".join(l)
212         self.pid=os.spawnvp(os.P_NOWAIT, l[0], l)
213         time.sleep(10)
214         signal.alarm(2400)
215         signal.signal(signal.SIGALRM, self.timeout)
216         try:
217             self.run()
218         finally:
219             signal.signal(signal.SIGALRM, signal.SIG_DFL)
220             os.kill(self.pid,15)
221             time.sleep(10)
222
223     def ssh(self,cmd):
224         l=['ssh','-o','UserKnownHostsFile=/dev/null','-o','StrictHostKeyChecking=no','-p','10022','-i',self.ssh_key,'%s@127.0.0.1'%self.login,cmd]
225         system(l)
226
227     def rsync(self,args,options='--delete --exclude .bzrignore'):
228         cmd ='rsync -rt -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 10022 -i %s" %s %s' % (self.ssh_key, options, args)
229         system(cmd)
230
231     def run(self):
232         pass
233
234 class KVMWinBuildExe(KVM):
235     def run(self):
236         with open(join(self.o.build_dir, 'setup/win32/Makefile.version'), 'w') as f:
237             f.write("VERSION=%s\n" % self.o.version_full)
238         with open(join(self.o.build_dir, 'setup/win32/Makefile.python'), 'w') as f:
239             f.write("PYTHON_VERSION=%s\n" % self.o.vm_winxp_python_version.replace('.', ''))
240
241         self.ssh("mkdir -p build")
242         self.rsync('%s/ %s@127.0.0.1:build/server/' % (self.o.build_dir, self.login))
243         self.ssh("cd build/server/setup/win32;time make allinone;")
244         self.rsync('%s@127.0.0.1:build/server/setup/win32/release/ %s/' % (self.login, self.o.build_dir), '')
245         print "KVMWinBuildExe.run(): done"
246
247 class KVMWinTestExe(KVM):
248     def run(self):
249         # Cannot use o.version_full when the version is not correctly parsed
250         # (for instance, containing *rc* or *dev*)
251         setuppath = glob("%s/openerp-server-setup-*.exe" % self.o.build_dir)[0]
252         setupfile = setuppath.split('/')[-1]
253         setupversion = setupfile.split('openerp-server-setup-')[1].split('.exe')[0]
254
255         self.rsync('"%s" %s@127.0.0.1:' % (setuppath, self.login))
256         self.ssh("TEMP=/tmp ./%s /S" % setupfile)
257         self.ssh('PGPASSWORD=openpgpwd /cygdrive/c/"Program Files"/"Odoo %s"/PostgreSQL/bin/createdb.exe -e -U openpg mycompany' % setupversion)
258         self.ssh('/cygdrive/c/"Program Files"/"Odoo %s"/server/openerp-server.exe -d mycompany -i base --stop-after-init' % setupversion)
259         self.ssh('net start odoo-server-8.0')
260         _rpc_count_modules(port=18069)
261
262 #----------------------------------------------------------
263 # Stage: building
264 #----------------------------------------------------------
265 def _prepare_build_dir(o):
266     cmd = ['rsync', '-a', '--exclude', '.git', '--exclude', '*.pyc', '--exclude', '*.pyo']
267     system(cmd + ['%s/' % o.odoo_dir, o.build_dir])
268     for i in glob(join(o.build_dir, 'addons/*')):
269         shutil.move(i, join(o.build_dir, 'openerp/addons'))
270
271 def build_tgz(o):
272     system(['python2', 'setup.py', '--quiet', 'sdist'], o.build_dir)
273     system(['cp', glob('%s/dist/odoo-*.tar.gz' % o.build_dir)[0], '%s/odoo.tar.gz' % o.build_dir])
274
275 def build_deb(o):
276     deb = pexpect.spawn('dpkg-buildpackage -rfakeroot -k%s' % GPGID, cwd=o.build_dir)
277     deb.logfile = stdout
278     deb.expect_exact('Enter passphrase: ', timeout=1200)
279     deb.send(GPGPASSPHRASE + '\r\n')
280     deb.expect_exact('Enter passphrase: ')
281     deb.send(GPGPASSPHRASE + '\r\n')
282     deb.expect(pexpect.EOF)
283     system(['mv', glob('%s/../odoo_*.deb' % o.build_dir)[0], '%s' % o.build_dir])
284     system(['mv', glob('%s/../odoo_*.dsc' % o.build_dir)[0], '%s' % o.build_dir])
285     system(['mv', glob('%s/../odoo_*_amd64.changes' % o.build_dir)[0], '%s' % o.build_dir])
286     system(['mv', glob('%s/../odoo_*.tar.gz' % o.build_dir)[0], '%s' % o.build_dir])
287
288 def build_rpm(o):
289     system(['python2', 'setup.py', '--quiet', 'bdist_rpm'], o.build_dir)
290     system(['cp', glob('%s/dist/odoo-*.noarch.rpm' % o.build_dir)[0], '%s/odoo.noarch.rpm' % o.build_dir])
291
292 def build_exe(o):
293     KVMWinBuildExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
294     system(['cp', glob('%s/openerp*.exe' % o.build_dir)[0], '%s/odoo.exe' % o.build_dir])
295
296 #----------------------------------------------------------
297 # Stage: testing
298 #----------------------------------------------------------
299 def test_tgz(o):
300     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
301         wheezy.release = 'odoo.tar.gz'
302         wheezy.system('apt-get update -qq && apt-get upgrade -qq -y')
303         wheezy.system("apt-get install postgresql python-dev postgresql-server-dev-all python-pip build-essential libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libssl-dev libjpeg-dev -y")
304         wheezy.system("service postgresql start")
305         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
306         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
307         wheezy.system('pip install -r /opt/release/requirements.txt')
308         wheezy.system('/usr/local/bin/pip install /opt/release/%s' % wheezy.release)
309         wheezy.system("useradd --system --no-create-home odoo")
310         wheezy.system('su postgres -s /bin/bash -c "createuser -s odoo"')
311         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
312         wheezy.system('mkdir /var/lib/odoo')
313         wheezy.system('chown odoo:odoo /var/lib/odoo')
314         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany -i base --stop-after-init"')
315         wheezy.system('su odoo -s /bin/bash -c "odoo.py --addons-path=/usr/local/lib/python2.7/dist-packages/openerp/addons -d mycompany &"')
316
317 def test_deb(o):
318     with docker('debian:stable', o.build_dir, o.pub) as wheezy:
319         wheezy.release = '*.deb'
320         wheezy.system('/usr/bin/apt-get update -qq && /usr/bin/apt-get upgrade -qq -y')
321         wheezy.system("apt-get install postgresql -y")
322         wheezy.system("service postgresql start")
323         wheezy.system('su postgres -s /bin/bash -c "pg_dropcluster --stop 9.1 main"')
324         wheezy.system('su postgres -s /bin/bash -c "pg_createcluster --start -e UTF-8 9.1 main"')
325         wheezy.system('su postgres -s /bin/bash -c "createdb mycompany"')
326         wheezy.system('/usr/bin/dpkg -i /opt/release/%s' % wheezy.release)
327         wheezy.system('/usr/bin/apt-get install -f -y')
328         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
329         wheezy.system('su odoo -s /bin/bash -c "odoo.py -c /etc/odoo/openerp-server.conf -d mycompany &"')
330
331 def test_rpm(o):
332     with docker('centos:centos7', o.build_dir, o.pub) as centos7:
333         centos7.release = 'odoo.noarch.rpm'
334         # Dependencies
335         centos7.system('yum install -d 0 -e 0 epel-release -y')
336         centos7.system('yum update -d 0 -e 0 -y')
337         # Manual install/start of postgres
338         centos7.system('yum install -d 0 -e 0 postgresql postgresql-server postgresql-libs postgresql-contrib postgresql-devel -y')
339         centos7.system('mkdir -p /var/lib/postgres/data')
340         centos7.system('chown -R postgres:postgres /var/lib/postgres/data')
341         centos7.system('chmod 0700 /var/lib/postgres/data')
342         centos7.system('su postgres -c "initdb -D /var/lib/postgres/data -E UTF-8"')
343         centos7.system('cp /usr/share/pgsql/postgresql.conf.sample /var/lib/postgres/data/postgresql.conf')
344         centos7.system('su postgres -c "/usr/bin/pg_ctl -D /var/lib/postgres/data start"')
345         centos7.system('sleep 5')
346         centos7.system('su postgres -c "createdb mycompany"')
347         # Odoo install
348         centos7.system('yum install -d 0 -e 0 /opt/release/%s -y' % centos7.release)
349         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany -i base --stop-after-init"')
350         centos7.system('su odoo -s /bin/bash -c "openerp-server -c /etc/odoo/openerp-server.conf -d mycompany &"')
351
352 def test_exe(o):
353     KVMWinTestExe(o, o.vm_winxp_image, o.vm_winxp_ssh_key, o.vm_winxp_login).start()
354
355 #---------------------------------------------------------
356 # Generates Packages, Sources and Release files of debian package
357 #---------------------------------------------------------
358 def gen_deb_package(o, published_files):
359     # Executes command to produce file_name in path, and moves it to o.pub/deb
360     def _gen_file(o, (command, file_name), path):
361         cur_tmp_file_path = os.path.join(path, file_name)
362         with open(cur_tmp_file_path, 'w') as out:
363             subprocess.call(command, stdout=out, cwd=path)
364         system(['cp', cur_tmp_file_path, os.path.join(o.pub, 'deb', file_name)])
365
366     # Copy files to a temp directory (required because the working directory must contain only the
367     # files of the last release)
368     temp_path = tempfile.mkdtemp(suffix='debPackages')
369     for pub_file_path in published_files:
370         system(['cp', pub_file_path, temp_path])
371
372     commands = [
373         (['dpkg-scanpackages', '.'], "Packages"),  # Generate Packages file
374         (['dpkg-scansources', '.'], "Sources"),  # Generate Sources file
375         (['apt-ftparchive', 'release', '.'], "Release")  # Generate Release file
376     ]
377     # Generate files
378     for command in commands:
379         _gen_file(o, command, temp_path)
380     # Remove temp directory
381     shutil.rmtree(temp_path)
382
383     # Generate Release.gpg (= signed Release)
384     # Options -abs: -a (Create ASCII armored output), -b (Make a detach signature), -s (Make a signature)
385     subprocess.call(['gpg', '--default-key', GPGID, '--passphrase', GPGPASSPHRASE, '--yes', '-abs', '--no-tty', '-o', 'Release.gpg', 'Release'], cwd=os.path.join(o.pub, 'deb'))
386
387 #---------------------------------------------------------
388 # Generates an RPM repo
389 #---------------------------------------------------------
390 def gen_rpm_repo(o, file_name):
391     # Sign the RPM
392     rpmsign = pexpect.spawn('/bin/bash', ['-c', 'rpm --resign %s' % file_name], cwd=os.path.join(o.pub, 'rpm'))
393     rpmsign.expect_exact('Enter pass phrase: ')
394     rpmsign.send(GPGPASSPHRASE + '\r\n')
395     rpmsign.expect(pexpect.EOF)
396
397     # Removes the old repodata
398     subprocess.call(['rm', '-rf', os.path.join(o.pub, 'rpm', 'repodata')])
399
400     # Copy files to a temp directory (required because the working directory must contain only the
401     # files of the last release)
402     temp_path = tempfile.mkdtemp(suffix='rpmPackages')
403     subprocess.call(['cp', file_name, temp_path])
404
405     subprocess.call(['createrepo', temp_path])  # creates a repodata folder in temp_path
406     subprocess.call(['cp', '-r', os.path.join(temp_path, "repodata"), os.path.join(o.pub, 'rpm')])
407
408     # Remove temp directory
409     shutil.rmtree(temp_path)
410
411 #----------------------------------------------------------
412 # Options and Main
413 #----------------------------------------------------------
414 def options():
415     op = optparse.OptionParser()
416     root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
417     build_dir = "%s-%s" % (root, timestamp)
418
419     op.add_option("-b", "--build-dir", default=build_dir, help="build directory (%default)", metavar="DIR")
420     op.add_option("-p", "--pub", default=None, help="pub directory (%default)", metavar="DIR")
421     op.add_option("", "--no-testing", action="store_true", help="don't test the builded packages")
422     op.add_option("-v", "--version", default='8.0', help="version (%default)")
423
424     op.add_option("", "--no-debian", action="store_true", help="don't build the debian package")
425     op.add_option("", "--no-rpm", action="store_true", help="don't build the rpm package")
426     op.add_option("", "--no-tarball", action="store_true", help="don't build the tarball")
427     op.add_option("", "--no-windows", action="store_true", help="don't build the windows package")
428
429     # Windows VM
430     op.add_option("", "--vm-winxp-image", default='/home/odoo/vm/winxp27/winxp27.vdi', help="%default")
431     op.add_option("", "--vm-winxp-ssh-key", default='/home/odoo/vm/winxp27/id_rsa', help="%default")
432     op.add_option("", "--vm-winxp-login", default='Naresh', help="Windows login (%default)")
433     op.add_option("", "--vm-winxp-python-version", default='2.7', help="Windows Python version installed in the VM (default: %default)")
434
435     (o, args) = op.parse_args()
436     # derive other options
437     o.odoo_dir = root
438     o.pkg = join(o.build_dir, 'pkg')
439     o.version_full = '%s-%s' % (o.version, timestamp)
440     o.work = join(o.build_dir, 'openerp-%s' % o.version_full)
441     o.work_addons = join(o.work, 'openerp', 'addons')
442
443     return o
444
445 def main():
446     o = options()
447     _prepare_build_dir(o)
448     try:
449         if not o.no_tarball:
450             build_tgz(o)
451             try:
452                 if not o.no_testing:
453                     test_tgz(o)
454                 published_files = publish(o, 'tarball', ['odoo.tar.gz'])
455             except Exception, e:
456                 print("Won't publish the tgz release.\n Exception: %s" % str(e))
457         if not o.no_debian:
458             build_deb(o)
459             try:
460                 if not o.no_testing:
461                     test_deb(o)
462
463                 to_publish = []
464                 to_publish.append(glob("%s/odoo_*.deb" % o.build_dir)[0])
465                 to_publish.append(glob("%s/odoo_*.dsc" % o.build_dir)[0])
466                 to_publish.append(glob("%s/odoo_*.changes" % o.build_dir)[0])
467                 to_publish.append(glob("%s/odoo_*.tar.gz" % o.build_dir)[0])
468                 published_files = publish(o, 'debian', to_publish)
469                 gen_deb_package(o, published_files)
470             except Exception, e:
471                 print("Won't publish the deb release.\n Exception: %s" % str(e))
472         if not o.no_rpm:
473             build_rpm(o)
474             try:
475                 if not o.no_testing:
476                     test_rpm(o)
477                 published_files = publish(o, 'redhat', ['odoo.noarch.rpm'])
478                 gen_rpm_repo(o, published_files[0])
479             except Exception, e:
480                 print("Won't publish the rpm release.\n Exception: %s" % str(e))
481         if not o.no_windows:
482             build_exe(o)
483             try:
484                 if not o.no_testing:
485                     test_exe(o)
486                 published_files = publish(o, 'windows', ['odoo.exe'])
487             except Exception, e:
488                 print("Won't publish the exe release.\n Exception: %s" % str(e))
489     except:
490         pass
491     finally:
492         shutil.rmtree(o.build_dir)
493         print('Build dir %s removed' % o.build_dir)
494
495         if not o.no_testing:
496             system("docker rm -f `docker ps -a | awk '{print $1 }'` 2>>/dev/null")
497             print('Remaining dockers removed')
498
499
500 if __name__ == '__main__':
501     main()