cpshousekeeping

Revision 53212, 17.7 kB (checked in by madarche, 7 months ago)
Removed a now unused parameter.
Property svn:eol-style set to `native` Property svn:executable set to ``* Property svn:keywords set to `Id`

Line
1	#!/usr/bin/python
2	#
3	# (C) Copyright 2005-2008 Nuxeo SAS <http://nuxeo.com>
4	# Authors:
5	# M.-A. Darche <madarche@nuxeo.com>
6	# Olivier Grisel <og@nuxeo.com>
7	# Julien Anguenot <ja@nuxeo.com>
8	#
9	# This program is free software; you can redistribute it and/or modify
10	# it under the terms of the GNU General Public License version 2 as published
11	# by the Free Software Foundation.
12	#
13	# This program is distributed in the hope that it will be useful,
14	# but WITHOUT ANY WARRANTY; without even the implied warranty of
15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	# GNU General Public License for more details.
17	#
18	# You should have received a copy of the GNU General Public License
19	# along with this program; if not, write to the Free Software
20	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
21	# 02111-1307, USA.
22	#
23	# $Id$
24	"""A script to perform various housekeeping tasks on CPS by calling Zope.
25
26	This script should be run directly against the server running Zope through HTTP
27	and locally, ie not called through Apache httpd and run on localhost.
28
29	A good practice is to call cpshousekeeping from another script, for example
30	a zopecpshousekeeping shell script, so that the user name and password for
31	connecting to Zope are never displayed in the email notifications sent by the
32	cron daemon.
33
34	Depending on your needs, call this script in /etc/cron.daily/,
35	/etc/cron.weekly/, /etc/cron.monthly/ or as an item in a user crontab through
36	the "crontab -e" command.
37
38	For example start the zopecpshousekeeping script every night at 03h59:
39
40	59 3 * * * zopecpshousekeeping
41
42	With the zopecpshousekeeping shell script containing the full command:
43
44	#!/bin/sh
45
46	/usr/local/sbin/cpshousekeeping --verbose --user admin --password 'xxx' --host localhost --port 8080 --instance-id cps --purge-repository --purge-localroles-force --pack-zodb --zodbfile /home/zope/cps/var/Data.fs --backupdir /var/backups/zodb/www.mysite.net
47
48	"""
49
50	import sys
51	import os
52	import gzip
53	import base64
54	from urllib import urlencode
55	import urllib2
56	import re
57	from time import strftime, gmtime
58	from optparse import OptionParser
59
60	DEFAULT_HOST_NAME = 'localhost'
61	DEFAULT_HOST_PORT = 8080
62	DEFAULT_INSTANCE_ID = 'cps'
63	DEFAULT_USER_NAME = 'admin'
64	DEFAULT_USER_PASSWORD = 'admin'
65	DEFAULT_ARCHIVED_REVISION_COUNT = 0
66	DEFAULT_HISTORY_DAYS = 0
67	DEFAULT_ZODB_PATH = '/usr/local/zope/instance/cps/var/Data.fs'
68	DEFAULT_ZODB_BACKUP_DIR_PATH = '/var/backups/zodb'
69	DEFAULT_ZODB_BACKUPS_KEEP_COUNT = 7
70	DEFAULT_NOTIFICATION_FREQ = None
71
72	ZODB_PACKING_URL_PATTERN = 'http://%s:%s/Control_Panel/Database/manage_pack'
73	PURGE_LOCALROLES_URL_PATTERN = 'http://%s:%s/%s/portal_membership/manage_purgeLocalRoles'
74	PURGE_LOCALROLES_FORCED_URL_PATTERN = 'http://%s:%s/%s/portal_membership/manage_purgeLocalRoles'
75	PURGE_DELETED_REVISIONS_URL_PATTERN = 'http://%s:%s/%s/portal_repository/manage_purgeDeletedRevisions'
76	PURGE_ARCHIVED_REVISIONS_URL_PATTERN = 'http://%s:%s/%s/portal_repository/manage_purgeArchivedRevisions'
77	TIME_FORMAT = '%Y-%m-%d_%H:%M'
78	# This regexp matches file names of the form
79	# YYYY-MM-dd_hh:mm-xxxxxxxxxx
80	# Examples:
81	# 2007-09-17_11:24-Data.fs
82	# 2007-09-17_11:24-Data.fs.gz
83	ZODB_BACKUP_FILENAME_REGEXP = re.compile(r'\d+-\d{2}-\d{2}_\d{2}:\d{2}-.+')
84	SEND_NOTIFICATIONS_PATTERN = 'http://%s:%s/%s/cps_subscriptions_schedule_notifications?subscription_mode=%s'
85
86
87	def execArgs():
88	"""Analyze command line arguments.
89	"""
90	usage = """Usage: %prog [options]
91
92	Example:
93	%prog --verbose --user admin --password 'xxx' --host localhost --port 8080 --instance-id cps --purge-repository --purge-localroles-force --pack-zodb --zodbfile /home/zope/cps/var/Data.fs --backupdir /var/backups/zodb/www.mysite.net
94	"""
95	parser = OptionParser(usage=usage)
96
97	parser.add_option('-v', '--verbose',
98	action='store_true',
99	dest='verbose',
100	default=False,
101	help="Print additional information to stdout.")
102
103	parser.add_option('-n', '--host',
104	action='store',
105	dest='host_name',
106	type='string',
107	metavar='NAME',
108	default=DEFAULT_HOST_NAME,
109	help="Use NAME for the server to connect to. "
110	"Defaults to '%s'." % DEFAULT_HOST_NAME)
111
112	parser.add_option('-p', '--port',
113	action='store',
114	dest='host_port',
115	type='int',
116	metavar='NUMBER',
117	default=DEFAULT_HOST_PORT,
118	help="Use NUMBER for the server port to use. "
119	"Defaults to %s." % DEFAULT_HOST_PORT)
120
121	parser.add_option('-i', '--instance-id',
122	action='store',
123	dest='instance_id',
124	type='string',
125	metavar='ID',
126	default=DEFAULT_INSTANCE_ID,
127	help="Use ID for CPS instance id to use. "
128	"Defaults to '%s'." % DEFAULT_INSTANCE_ID)
129
130	parser.add_option('-u', '--user',
131	action='store',
132	dest='user_name',
133	type='string',
134	metavar='NAME',
135	default=DEFAULT_USER_NAME,
136	help="Use NAME for the user name to Zope. "
137	"Defaults to '%s'." % DEFAULT_USER_NAME)
138
139	parser.add_option('-w', '--password',
140	action='store',
141	dest='user_password',
142	type='string',
143	metavar='PASSWORD',
144	default=DEFAULT_USER_PASSWORD,
145	help="Use PASSWORD for the password to Zope. "
146	"Defaults to '%s'." % DEFAULT_USER_PASSWORD)
147
148	parser.add_option('-r', '--purge-repository',
149	action='store_true',
150	dest='purge_repository',
151	default=False,
152	help="Purge the repository to remove orphan documents.")
153
154	parser.add_option('-l', '--purge-localroles',
155	action='store_true',
156	dest='purge_localroles',
157	default=False,
158	help="Purge the localroles of deleted members "
159	"recorded as such.")
160
161	parser.add_option('-L', '--purge-localroles-force',
162	action='store_true',
163	dest='purge_localroles_force',
164	default=False,
165	help="Purge the localroles of deleted members.")
166
167	parser.add_option('--purge-archived-revisions',
168	action='store_true',
169	dest='purge_archived_revisions',
170	default=False,
171	help="Purge the archived revisions of documents.")
172
173	parser.add_option('--archived-revisions',
174	action='store',
175	dest='archived_revisions',
176	type='int',
177	metavar='NUMBER',
178	default=DEFAULT_ARCHIVED_REVISION_COUNT,
179	help="Use NUMBER for the number of archives "
180	"to keep for each document. "
181	"Defaults to %s." % DEFAULT_HISTORY_DAYS)
182
183	parser.add_option('-P', '--pack-zodb',
184	action='store_true',
185	dest='pack_zodb',
186	default=False,
187	help="Pack the ZODB.")
188
189	parser.add_option('-d', '--days',
190	action='store',
191	dest='days',
192	type='float',
193	metavar='NUMBER',
194	default=DEFAULT_HISTORY_DAYS,
195	help="Use NUMBER for the days to keep in history. "
196	"Defaults to %s." % DEFAULT_HISTORY_DAYS)
197
198	parser.add_option('-z', '--zodbfile',
199	action='store',
200	dest='zodbfile_path',
201	type='string',
202	metavar='FILE',
203	default=DEFAULT_ZODB_PATH,
204	help="The FILE path to the ZODB. "
205	"The default is %s." % DEFAULT_ZODB_PATH)
206
207	parser.add_option('-b', '--backup',
208	action='store_true',
209	dest='backup',
210	default=False,
211	help="Backup the ZODB that has just been packed "
212	"""using the "cp" command.""")
213
214	parser.add_option('-k', '--backupdir',
215	action='store',
216	dest='backupdir_path',
217	type='string',
218	metavar='FILE',
219	default=DEFAULT_ZODB_BACKUP_DIR_PATH,
220	help="The FILE path of the directory used for storing "
221	"ZODB backups. "
222	"The default is %s." % DEFAULT_ZODB_BACKUP_DIR_PATH)
223
224	parser.add_option('--nocompress',
225	action='store_true',
226	dest='nocompress',
227	default=False,
228	help="Don't compress the ZODB backups.")
229
230	parser.add_option('-B', '--keep-backups-count',
231	action='store',
232	dest='backups_keep_count',
233	type='int',
234	metavar='NUMBER',
235	default=DEFAULT_ZODB_BACKUPS_KEEP_COUNT,
236	help="The NUMBER of ZODB backups to keep. "
237	"The default is %s." % DEFAULT_ZODB_BACKUPS_KEEP_COUNT)
238
239	parser.add_option('-s', '--send-notifications',
240	action='store',
241	dest='notifications',
242	type='choice',
243	metavar='FREQ',
244	choices=['daily','weekly', 'monthly'],
245	default=DEFAULT_NOTIFICATION_FREQ,
246	help="Send email notifications of frequence FREQ of "
247	"[daily\|weekly\|monthly]. Defaults to %s." %
248	str(DEFAULT_NOTIFICATION_FREQ))
249
250	(options, args) = parser.parse_args()
251	global verbose
252	verbose = options.verbose
253
254	if options.purge_repository:
255	log("\nPurging the document repository of host %s ..." %
256	options.host_name)
257	url = PURGE_DELETED_REVISIONS_URL_PATTERN % (options.host_name,
258	options.host_port,
259	options.instance_id)
260	postUrl(url, options.user_name, options.user_password)
261	log("Successfully purged repository of host %s" % options.host_name)
262
263	if options.purge_localroles:
264	log("\nPurging the local roles of host %s ..." % options.host_name)
265	url = PURGE_LOCALROLES_URL_PATTERN % (options.host_name,
266	options.host_port,
267	options.instance_id)
268	postUrl(url, options.user_name, options.user_password)
269	log("Successfully purged localroles of host %s" % options.host_name)
270
271	if options.purge_localroles_force:
272	log("\nPurging the local roles (forced) of host %s ..."
273	% options.host_name)
274	url = PURGE_LOCALROLES_FORCED_URL_PATTERN % (options.host_name,
275	options.host_port,
276	options.instance_id)
277	postUrl(url, options.user_name, options.user_password,
278	params={'lazy:boolean': False})
279	log("Successfully purged localroles (forced) of host %s"
280	% options.host_name)
281
282	if options.purge_archived_revisions:
283	log("\nPurging archived revisions of documents of host %s "
284	"(keeping no more than %s archives per doc) ..."
285	% (options.host_name, options.archived_revisions))
286	url = PURGE_ARCHIVED_REVISIONS_URL_PATTERN % (options.host_name,
287	options.host_port,
288	options.instance_id)
289	postUrl(url, options.user_name, options.user_password,
290	params={'keep_max': options.archived_revisions})
291	log("Successfully purged archived revisions of documents of host %s"
292	% options.host_name)
293
294	if options.notifications:
295	log("\nSending notifications of host %s ..." % options.host_name)
296	url = SEND_NOTIFICATIONS_PATTERN % (options.host_name,
297	options.host_port,
298	options.instance_id,
299	options.notifications)
300	postUrl(url, options.user_name, options.user_password)
301	log("Successfully sent notifications of host %s" % options.host_name)
302
303	if options.pack_zodb:
304	packZodb(options.zodbfile_path, options.days,
305	options.host_name, options.host_port,
306	options.user_name, options.user_password)
307
308	if options.backup:
309	backupZodb(options.zodbfile_path, options.backupdir_path,
310	not options.nocompress, options.backups_keep_count)
311
312
313	def packZodb(zodb_path, days, host_name, host_port, user_name, user_password):
314	"""TODO : packZodb should WARN and not perform this action if there isn't
315	enough space left on the device where the ZODB is.
316	The method should check if there is 2x the size of the ZODB left.
317	"""
318	log("\nPacking ZODB %s of host %s ..." % (zodb_path, host_name))
319	if verbose:
320	zodb_size = os.path.getsize(zodb_path)
321	log("ZODB size is %s" % getHumanReadableSize(zodb_size),
322	increment=1)
323	url = ZODB_PACKING_URL_PATTERN % (host_name, host_port)
324	postUrl(url, user_name, user_password, params={'days:float': days})
325	if verbose:
326	zodb_size = os.path.getsize(zodb_path)
327	log("ZODB size now is %s" % getHumanReadableSize(zodb_size),
328	increment=1)
329	log("Successfully packed ZODB of host %s" % host_name)
330
331	def backupZodb(zodb_path, backupdir_path, compress=True, backups_keep_count=0):
332	"""TODO : backupZodb should WARN and not perform this action if there isn't
333	enough space left on target device. But unfortunately this requires parsing
334	of platform specific information and there isn't any native python-way to
335	do this yet.
336	"""
337	log("\nChecking for backups ...")
338	time_string = strftime(TIME_FORMAT, gmtime())
339	file_name = '%s-%s' % (time_string, os.path.basename(zodb_path))
340	zodb_backup_path = os.path.join(backupdir_path, file_name)
341	command = "cp -p %s %s" % (zodb_path, zodb_backup_path)
342	log("command = %s" % command, increment=1)
343	os.system(command)
344	if compress:
345	compressFile(zodb_backup_path)
346
347	if backups_keep_count > 0:
348	log("Checking for potential backups to remove, "
349	"keeping only the %s ones ..." % backups_keep_count, increment=1)
350	file_names = os.listdir(backupdir_path)
351	file_names = [x for x in file_names
352	if ZODB_BACKUP_FILENAME_REGEXP.match(x)]
353	file_names.sort()
354	file_names_to_delete = file_names[:-backups_keep_count]
355	for file_name in file_names_to_delete:
356	file_path = os.path.join(backupdir_path, file_name)
357	log("Removing = %s ..." % file_path, increment=1)
358	os.remove(file_path)
359	log("Successful backup of the ZODB")
360
361
362	def compressFile(file_path):
363	"""Compress a file using GZIP.
364	"""
365	log("Compressing %s ..." % file_path)
366	f_in = open(file_path, 'rb')
367	gzip_file_path = file_path + '.gz'
368	f_out = gzip.open(gzip_file_path, 'wb')
369	f_out.writelines(f_in)
370	f_out.close()
371	os.remove(file_path)
372	log("Compressing done : %s" % gzip_file_path)
373
374
375	def postUrl(url, username, password, params={}):
376	"""Call urllib2.urlopen with forced HTTP Basic Auth header.
377	"""
378	request = urllib2.Request(url)
379	base64string = base64.encodestring('%s:%s' % (username, password))[:-1]
380	request.add_header('Authorization', 'Basic %s' % base64string)
381	if params:
382	params_encoded = urlencode(params)
383	request.add_data(params_encoded)
384	else:
385	# Adding this to make sure the request is a POST request,
386	# because this method need to be POST because it modifies
387	# the state of the application.
388	request.add_data("This is a POST method")
389	try:
390	urllib2.urlopen(request)
391	except urllib2.HTTPError, e:
392	if e.code == 401:
393	log('Invalid credentials on %s, aborting' % url, True)
394
395	elif e.code == 400:
396	# XXX: manage_purgeLocalRoles returns code 400
397	return
398	else:
399	log('Error %d on %s, aborting' % (e.code, url), True)
400	sys.exit(1)
401	except IOError:
402	log('Unable to open %s, aborting' % url, True)
403	sys.exit(1)
404
405
406	def getHumanReadableSize(octet_size):
407	"""Returns a string that is a human readable file size.
408	"""
409	mega = 1024*1024
410	kilo = 1024
411
412	if octet_size is None or octet_size <= 0:
413	return "0"
414	elif octet_size >= mega:
415	if octet_size == mega:
416	return "1M"
417	else:
418	msize = float(octet_size/float(mega))
419	msize = float('%.02f' % msize)
420	return "%sM" % msize
421	elif octet_size >= kilo:
422
423	if octet_size == kilo:
424	return "1K"
425	else:
426	ksize = float(octet_size/float(kilo))
427	ksize = float('%.02f' % ksize)
428	return "%sK" % ksize
429	else:
430	return str(octet_size)
431
432
433	def getSpaceLeftOnDevice():
434	"""TODO
435	$ df -h \| grep '^/dev/' \| awk '{print $4}'
436	"""
437	pass
438
439
440	def log(message, force=False, increment=0):
441	"""Log the given message to stderr.
442	"""
443	message = ' ' * increment + message
444	if force or verbose:
445	print >> sys.stderr, message
446
447
448	# Shell conversion
449	if __name__ == '__main__':
450	execArgs()
451

Note: See TracBrowser for help on using the browser.

root/CPS3/products/CPSUtil/trunk/bin/cpshousekeeping

Download in other formats: