1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41 """
42 Provides general-purpose utilities.
43
44 @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList,
45 RegexList, _Vertex, DirectedGraph, PathResolverSingleton,
46 sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine,
47 resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice,
48 deriveDayOfWeek, isStartOfWeek, buildNormalizedPath,
49 ISO_SECTOR_SIZE, BYTES_PER_SECTOR,
50 BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE,
51 SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY,
52 UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS
53
54 @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes.
55 @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector.
56 @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB).
57 @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB).
58 @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB).
59 @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB).
60 @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB).
61 @var SECONDS_PER_MINUTE: Number of seconds per minute.
62 @var MINUTES_PER_HOUR: Number of minutes per hour.
63 @var HOURS_PER_DAY: Number of hours per day.
64 @var SECONDS_PER_DAY: Number of seconds per day.
65 @var UNIT_BYTES: Constant representing the byte (B) unit for conversion.
66 @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion.
67 @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion.
68 @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion.
69 @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion.
70
71 @author: Kenneth J. Pronovici <pronovic@ieee.org>
72 """
73
74
75
76
77
78
79 import sys
80 import math
81 import os
82 import re
83 import time
84 import logging
85 import string
86 from subprocess import Popen, STDOUT, PIPE
87
88 try:
89 import pwd
90 import grp
91 _UID_GID_AVAILABLE = True
92 except ImportError:
93 _UID_GID_AVAILABLE = False
94
95 from CedarBackup2.release import VERSION, DATE
96
97
98
99
100
101
102 logger = logging.getLogger("CedarBackup2.log.util")
103 outputLogger = logging.getLogger("CedarBackup2.output")
104
105 ISO_SECTOR_SIZE = 2048.0
106 BYTES_PER_SECTOR = ISO_SECTOR_SIZE
107
108 BYTES_PER_KBYTE = 1024.0
109 KBYTES_PER_MBYTE = 1024.0
110 MBYTES_PER_GBYTE = 1024.0
111 BYTES_PER_MBYTE = BYTES_PER_KBYTE * KBYTES_PER_MBYTE
112 BYTES_PER_GBYTE = BYTES_PER_MBYTE * MBYTES_PER_GBYTE
113
114 SECONDS_PER_MINUTE = 60.0
115 MINUTES_PER_HOUR = 60.0
116 HOURS_PER_DAY = 24.0
117 SECONDS_PER_DAY = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY
118
119 UNIT_BYTES = 0
120 UNIT_KBYTES = 1
121 UNIT_MBYTES = 2
122 UNIT_GBYTES = 4
123 UNIT_SECTORS = 3
124
125 MTAB_FILE = "/etc/mtab"
126
127 MOUNT_COMMAND = [ "mount", ]
128 UMOUNT_COMMAND = [ "umount", ]
129
130 DEFAULT_LANGUAGE = "C"
131 LANG_VAR = "LANG"
132 LOCALE_VARS = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE",
133 "LC_CTYPE", "LC_IDENTIFICATION",
134 "LC_MEASUREMENT", "LC_MESSAGES",
135 "LC_MONETARY", "LC_NAME", "LC_NUMERIC",
136 "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ]
144
145 """
146 Class representing an "unordered list".
147
148 An "unordered list" is a list in which only the contents matter, not the
149 order in which the contents appear in the list.
150
151 For instance, we might be keeping track of set of paths in a list, because
152 it's convenient to have them in that form. However, for comparison
153 purposes, we would only care that the lists contain exactly the same
154 contents, regardless of order.
155
156 I have come up with two reasonable ways of doing this, plus a couple more
157 that would work but would be a pain to implement. My first method is to
158 copy and sort each list, comparing the sorted versions. This will only work
159 if two lists with exactly the same members are guaranteed to sort in exactly
160 the same order. The second way would be to create two Sets and then compare
161 the sets. However, this would lose information about any duplicates in
162 either list. I've decided to go with option #1 for now. I'll modify this
163 code if I run into problems in the future.
164
165 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__},
166 C{__le__} and C{__lt__} list methods to change the definition of the various
167 comparison operators. In all cases, the comparison is changed to return the
168 result of the original operation I{but instead comparing sorted lists}.
169 This is going to be quite a bit slower than a normal list, so you probably
170 only want to use it on small lists.
171 """
172
174 """
175 Definition of C{==} operator for this class.
176 @param other: Other object to compare to.
177 @return: True/false depending on whether C{self == other}.
178 """
179 if other is None:
180 return False
181 selfSorted = self[:]
182 otherSorted = other[:]
183 selfSorted.sort()
184 otherSorted.sort()
185 return selfSorted.__eq__(otherSorted)
186
188 """
189 Definition of C{!=} operator for this class.
190 @param other: Other object to compare to.
191 @return: True/false depending on whether C{self != other}.
192 """
193 if other is None:
194 return True
195 selfSorted = self[:]
196 otherSorted = other[:]
197 selfSorted.sort()
198 otherSorted.sort()
199 return selfSorted.__ne__(otherSorted)
200
202 """
203 Definition of S{>=} operator for this class.
204 @param other: Other object to compare to.
205 @return: True/false depending on whether C{self >= other}.
206 """
207 if other is None:
208 return True
209 selfSorted = self[:]
210 otherSorted = other[:]
211 selfSorted.sort()
212 otherSorted.sort()
213 return selfSorted.__ge__(otherSorted)
214
216 """
217 Definition of C{>} operator for this class.
218 @param other: Other object to compare to.
219 @return: True/false depending on whether C{self > other}.
220 """
221 if other is None:
222 return True
223 selfSorted = self[:]
224 otherSorted = other[:]
225 selfSorted.sort()
226 otherSorted.sort()
227 return selfSorted.__gt__(otherSorted)
228
230 """
231 Definition of S{<=} operator for this class.
232 @param other: Other object to compare to.
233 @return: True/false depending on whether C{self <= other}.
234 """
235 if other is None:
236 return False
237 selfSorted = self[:]
238 otherSorted = other[:]
239 selfSorted.sort()
240 otherSorted.sort()
241 return selfSorted.__le__(otherSorted)
242
244 """
245 Definition of C{<} operator for this class.
246 @param other: Other object to compare to.
247 @return: True/false depending on whether C{self < other}.
248 """
249 if other is None:
250 return False
251 selfSorted = self[:]
252 otherSorted = other[:]
253 selfSorted.sort()
254 otherSorted.sort()
255 return selfSorted.__lt__(otherSorted)
256
263
264 """
265 Class representing a list of absolute paths.
266
267 This is an unordered list.
268
269 We override the C{append}, C{insert} and C{extend} methods to ensure that
270 any item added to the list is an absolute path.
271
272 Each item added to the list is encoded using L{encodePath}. If we don't do
273 this, we have problems trying certain operations between strings and unicode
274 objects, particularly for "odd" filenames that can't be encoded in standard
275 ASCII.
276 """
277
279 """
280 Overrides the standard C{append} method.
281 @raise ValueError: If item is not an absolute path.
282 """
283 if not os.path.isabs(item):
284 raise ValueError("Not an absolute path: [%s]" % item)
285 list.append(self, encodePath(item))
286
287 - def insert(self, index, item):
288 """
289 Overrides the standard C{insert} method.
290 @raise ValueError: If item is not an absolute path.
291 """
292 if not os.path.isabs(item):
293 raise ValueError("Not an absolute path: [%s]" % item)
294 list.insert(self, index, encodePath(item))
295
297 """
298 Overrides the standard C{insert} method.
299 @raise ValueError: If any item is not an absolute path.
300 """
301 for item in seq:
302 if not os.path.isabs(item):
303 raise ValueError("Not an absolute path: [%s]" % item)
304 for item in seq:
305 list.append(self, encodePath(item))
306
313
314 """
315 Class representing a list containing only objects with a certain type.
316
317 This is an unordered list.
318
319 We override the C{append}, C{insert} and C{extend} methods to ensure that
320 any item added to the list matches the type that is requested. The
321 comparison uses the built-in C{isinstance}, which should allow subclasses of
322 of the requested type to be added to the list as well.
323
324 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a
325 CollectDir object."} if C{objectName} is C{"CollectDir"}.
326 """
327
328 - def __init__(self, objectType, objectName):
329 """
330 Initializes a typed list for a particular type.
331 @param objectType: Type that the list elements must match.
332 @param objectName: Short string containing the "name" of the type.
333 """
334 super(ObjectTypeList, self).__init__()
335 self.objectType = objectType
336 self.objectName = objectName
337
339 """
340 Overrides the standard C{append} method.
341 @raise ValueError: If item does not match requested type.
342 """
343 if not isinstance(item, self.objectType):
344 raise ValueError("Item must be a %s object." % self.objectName)
345 list.append(self, item)
346
347 - def insert(self, index, item):
348 """
349 Overrides the standard C{insert} method.
350 @raise ValueError: If item does not match requested type.
351 """
352 if not isinstance(item, self.objectType):
353 raise ValueError("Item must be a %s object." % self.objectName)
354 list.insert(self, index, item)
355
357 """
358 Overrides the standard C{insert} method.
359 @raise ValueError: If item does not match requested type.
360 """
361 for item in seq:
362 if not isinstance(item, self.objectType):
363 raise ValueError("All items must be %s objects." % self.objectName)
364 list.extend(self, seq)
365
366
367
368
369
370
371 -class RestrictedContentList(UnorderedList):
372
373 """
374 Class representing a list containing only object with certain values.
375
376 This is an unordered list.
377
378 We override the C{append}, C{insert} and C{extend} methods to ensure that
379 any item added to the list is among the valid values. We use a standard
380 comparison, so pretty much anything can be in the list of valid values.
381
382 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be
383 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}.
384
385 @note: This class doesn't make any attempt to trap for nonsensical
386 arguments. All of the values in the values list should be of the same type
387 (i.e. strings). Then, all list operations also need to be of that type
388 (i.e. you should always insert or append just strings). If you mix types --
389 for instance lists and strings -- you will likely see AttributeError
390 exceptions or other problems.
391 """
392
393 - def __init__(self, valuesList, valuesDescr, prefix=None):
394 """
395 Initializes a list restricted to containing certain values.
396 @param valuesList: List of valid values.
397 @param valuesDescr: Short string describing list of values.
398 @param prefix: Prefix to use in error messages (None results in prefix "Item")
399 """
400 super(RestrictedContentList, self).__init__()
401 self.prefix = "Item"
402 if prefix is not None: self.prefix = prefix
403 self.valuesList = valuesList
404 self.valuesDescr = valuesDescr
405
406 - def append(self, item):
407 """
408 Overrides the standard C{append} method.
409 @raise ValueError: If item is not in the values list.
410 """
411 if item not in self.valuesList:
412 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
413 list.append(self, item)
414
415 - def insert(self, index, item):
416 """
417 Overrides the standard C{insert} method.
418 @raise ValueError: If item is not in the values list.
419 """
420 if item not in self.valuesList:
421 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
422 list.insert(self, index, item)
423
424 - def extend(self, seq):
425 """
426 Overrides the standard C{insert} method.
427 @raise ValueError: If item is not in the values list.
428 """
429 for item in seq:
430 if item not in self.valuesList:
431 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
432 list.extend(self, seq)
433
440
441 """
442 Class representing a list containing only strings that match a regular expression.
443
444 If C{emptyAllowed} is passed in as C{False}, then empty strings are
445 explicitly disallowed, even if they happen to match the regular expression.
446 (C{None} values are always disallowed, since string operations are not
447 permitted on C{None}.)
448
449 This is an unordered list.
450
451 We override the C{append}, C{insert} and C{extend} methods to ensure that
452 any item added to the list matches the indicated regular expression.
453
454 @note: If you try to put values that are not strings into the list, you will
455 likely get either TypeError or AttributeError exceptions as a result.
456 """
457
458 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
459 """
460 Initializes a list restricted to containing certain values.
461 @param valuesRegex: Regular expression that must be matched, as a string
462 @param emptyAllowed: Indicates whether empty or None values are allowed.
463 @param prefix: Prefix to use in error messages (None results in prefix "Item")
464 """
465 super(RegexMatchList, self).__init__()
466 self.prefix = "Item"
467 if prefix is not None: self.prefix = prefix
468 self.valuesRegex = valuesRegex
469 self.emptyAllowed = emptyAllowed
470 self.pattern = re.compile(self.valuesRegex)
471
473 """
474 Overrides the standard C{append} method.
475 @raise ValueError: If item is None
476 @raise ValueError: If item is empty and empty values are not allowed
477 @raise ValueError: If item does not match the configured regular expression
478 """
479 if item is None or (not self.emptyAllowed and item == ""):
480 raise ValueError("%s cannot be empty." % self.prefix)
481 if not self.pattern.search(item):
482 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
483 list.append(self, item)
484
485 - def insert(self, index, item):
486 """
487 Overrides the standard C{insert} method.
488 @raise ValueError: If item is None
489 @raise ValueError: If item is empty and empty values are not allowed
490 @raise ValueError: If item does not match the configured regular expression
491 """
492 if item is None or (not self.emptyAllowed and item == ""):
493 raise ValueError("%s cannot be empty." % self.prefix)
494 if not self.pattern.search(item):
495 raise ValueError("%s is not valid [%s]" % (self.prefix, item))
496 list.insert(self, index, item)
497
499 """
500 Overrides the standard C{insert} method.
501 @raise ValueError: If any item is None
502 @raise ValueError: If any item is empty and empty values are not allowed
503 @raise ValueError: If any item does not match the configured regular expression
504 """
505 for item in seq:
506 if item is None or (not self.emptyAllowed and item == ""):
507 raise ValueError("%s cannot be empty." % self.prefix)
508 if not self.pattern.search(item):
509 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
510 list.extend(self, seq)
511
512
513
514
515
516
517 -class RegexList(UnorderedList):
518
519 """
520 Class representing a list of valid regular expression strings.
521
522 This is an unordered list.
523
524 We override the C{append}, C{insert} and C{extend} methods to ensure that
525 any item added to the list is a valid regular expression.
526 """
527
529 """
530 Overrides the standard C{append} method.
531 @raise ValueError: If item is not an absolute path.
532 """
533 try:
534 re.compile(item)
535 except re.error:
536 raise ValueError("Not a valid regular expression: [%s]" % item)
537 list.append(self, item)
538
539 - def insert(self, index, item):
540 """
541 Overrides the standard C{insert} method.
542 @raise ValueError: If item is not an absolute path.
543 """
544 try:
545 re.compile(item)
546 except re.error:
547 raise ValueError("Not a valid regular expression: [%s]" % item)
548 list.insert(self, index, item)
549
551 """
552 Overrides the standard C{insert} method.
553 @raise ValueError: If any item is not an absolute path.
554 """
555 for item in seq:
556 try:
557 re.compile(item)
558 except re.error:
559 raise ValueError("Not a valid regular expression: [%s]" % item)
560 for item in seq:
561 list.append(self, item)
562
563
564
565
566
567
568 -class _Vertex(object):
569
570 """
571 Represents a vertex (or node) in a directed graph.
572 """
573
575 """
576 Constructor.
577 @param name: Name of this graph vertex.
578 @type name: String value.
579 """
580 self.name = name
581 self.endpoints = []
582 self.state = None
583
585
586 """
587 Represents a directed graph.
588
589 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set
590 B{E} of vertex pairs or edges. In a directed graph, each edge also has an
591 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph}
592 object provides a way to construct a directed graph and execute a depth-
593 first search.
594
595 This data structure was designed based on the graphing chapter in
596 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>},
597 by Steven S. Skiena.
598
599 This class is intended to be used by Cedar Backup for dependency ordering.
600 Because of this, it's not quite general-purpose. Unlike a "general" graph,
601 every vertex in this graph has at least one edge pointing to it, from a
602 special "start" vertex. This is so no vertices get "lost" either because
603 they have no dependencies or because nothing depends on them.
604 """
605
606 _UNDISCOVERED = 0
607 _DISCOVERED = 1
608 _EXPLORED = 2
609
611 """
612 Directed graph constructor.
613
614 @param name: Name of this graph.
615 @type name: String value.
616 """
617 if name is None or name == "":
618 raise ValueError("Graph name must be non-empty.")
619 self._name = name
620 self._vertices = {}
621 self._startVertex = _Vertex(None)
622
624 """
625 Official string representation for class instance.
626 """
627 return "DirectedGraph(%s)" % self.name
628
630 """
631 Informal string representation for class instance.
632 """
633 return self.__repr__()
634
636 """
637 Definition of equals operator for this class.
638 @param other: Other object to compare to.
639 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
640 """
641
642 if other is None:
643 return 1
644 if self.name != other.name:
645 if self.name < other.name:
646 return -1
647 else:
648 return 1
649 if self._vertices != other._vertices:
650 if self._vertices < other._vertices:
651 return -1
652 else:
653 return 1
654 return 0
655
657 """
658 Property target used to get the graph name.
659 """
660 return self._name
661
662 name = property(_getName, None, None, "Name of the graph.")
663
665 """
666 Creates a named vertex.
667 @param name: vertex name
668 @raise ValueError: If the vertex name is C{None} or empty.
669 """
670 if name is None or name == "":
671 raise ValueError("Vertex name must be non-empty.")
672 vertex = _Vertex(name)
673 self._startVertex.endpoints.append(vertex)
674 self._vertices[name] = vertex
675
677 """
678 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex.
679 @param start: Name of start vertex.
680 @param finish: Name of finish vertex.
681 @raise ValueError: If one of the named vertices is unknown.
682 """
683 try:
684 startVertex = self._vertices[start]
685 finishVertex = self._vertices[finish]
686 startVertex.endpoints.append(finishVertex)
687 except KeyError, e:
688 raise ValueError("Vertex [%s] could not be found." % e)
689
691 """
692 Implements a topological sort of the graph.
693
694 This method also enforces that the graph is a directed acyclic graph,
695 which is a requirement of a topological sort.
696
697 A directed acyclic graph (or "DAG") is a directed graph with no directed
698 cycles. A topological sort of a DAG is an ordering on the vertices such
699 that all edges go from left to right. Only an acyclic graph can have a
700 topological sort, but any DAG has at least one topological sort.
701
702 Since a topological sort only makes sense for an acyclic graph, this
703 method throws an exception if a cycle is found.
704
705 A depth-first search only makes sense if the graph is acyclic. If the
706 graph contains any cycles, it is not possible to determine a consistent
707 ordering for the vertices.
708
709 @note: If a particular vertex has no edges, then its position in the
710 final list depends on the order in which the vertices were created in the
711 graph. If you're using this method to determine a dependency order, this
712 makes sense: a vertex with no dependencies can go anywhere (and will).
713
714 @return: Ordering on the vertices so that all edges go from left to right.
715
716 @raise ValueError: If a cycle is found in the graph.
717 """
718 ordering = []
719 for key in self._vertices:
720 vertex = self._vertices[key]
721 vertex.state = self._UNDISCOVERED
722 for key in self._vertices:
723 vertex = self._vertices[key]
724 if vertex.state == self._UNDISCOVERED:
725 self._topologicalSort(self._startVertex, ordering)
726 return ordering
727
729 """
730 Recursive depth first search function implementing topological sort.
731 @param vertex: Vertex to search
732 @param ordering: List of vertices in proper order
733 """
734 vertex.state = self._DISCOVERED
735 for endpoint in vertex.endpoints:
736 if endpoint.state == self._UNDISCOVERED:
737 self._topologicalSort(endpoint, ordering)
738 elif endpoint.state != self._EXPLORED:
739 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name))
740 if vertex.name is not None:
741 ordering.insert(0, vertex.name)
742 vertex.state = self._EXPLORED
743
750
751 """
752 Singleton used for resolving executable paths.
753
754 Various functions throughout Cedar Backup (including extensions) need a way
755 to resolve the path of executables that they use. For instance, the image
756 functionality needs to find the C{mkisofs} executable, and the Subversion
757 extension needs to find the C{svnlook} executable. Cedar Backup's original
758 behavior was to assume that the simple name (C{"svnlook"} or whatever) was
759 available on the caller's C{$PATH}, and to fail otherwise. However, this
760 turns out to be less than ideal, since for instance the root user might not
761 always have executables like C{svnlook} in its path.
762
763 One solution is to specify a path (either via an absolute path or some sort
764 of path insertion or path appending mechanism) that would apply to the
765 C{executeCommand()} function. This is not difficult to implement, but it
766 seem like kind of a "big hammer" solution. Besides that, it might also
767 represent a security flaw (for instance, I prefer not to mess with root's
768 C{$PATH} on the application level if I don't have to).
769
770 The alternative is to set up some sort of configuration for the path to
771 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or
772 whatever. This PathResolverSingleton aims to provide a good solution to the
773 mapping problem. Callers of all sorts (extensions or not) can get an
774 instance of the singleton. Then, they call the C{lookup} method to try and
775 resolve the executable they are looking for. Through the C{lookup} method,
776 the caller can also specify a default to use if a mapping is not found.
777 This way, with no real effort on the part of the caller, behavior can neatly
778 degrade to something equivalent to the current behavior if there is no
779 special mapping or if the singleton was never initialized in the first
780 place.
781
782 Even better, extensions automagically get access to the same resolver
783 functionality, and they don't even need to understand how the mapping
784 happens. All extension authors need to do is document what executables
785 their code requires, and the standard resolver configuration section will
786 meet their needs.
787
788 The class should be initialized once through the constructor somewhere in
789 the main routine. Then, the main routine should call the L{fill} method to
790 fill in the resolver's internal structures. Everyone else who needs to
791 resolve a path will get an instance of the class using L{getInstance} and
792 will then just call the L{lookup} method.
793
794 @cvar _instance: Holds a reference to the singleton
795 @ivar _mapping: Internal mapping from resource name to path.
796 """
797
798 _instance = None
799
801 """Helper class to provide a singleton factory method."""
810
811 getInstance = _Helper()
812
819
820 - def lookup(self, name, default=None):
821 """
822 Looks up name and returns the resolved path associated with the name.
823 @param name: Name of the path resource to resolve.
824 @param default: Default to return if resource cannot be resolved.
825 @return: Resolved path associated with name, or default if name can't be resolved.
826 """
827 value = default
828 if name in self._mapping.keys():
829 value = self._mapping[name]
830 logger.debug("Resolved command [%s] to [%s].", name, value)
831 return value
832
833 - def fill(self, mapping):
834 """
835 Fills in the singleton's internal mapping from name to resource.
836 @param mapping: Mapping from resource name to path.
837 @type mapping: Dictionary mapping name to path, both as strings.
838 """
839 self._mapping = { }
840 for key in mapping.keys():
841 self._mapping[key] = mapping[key]
842
843
844
845
846
847
848 -class Pipe(Popen):
849 """
850 Specialized pipe class for use by C{executeCommand}.
851
852 The L{executeCommand} function needs a specialized way of interacting
853 with a pipe. First, C{executeCommand} only reads from the pipe, and
854 never writes to it. Second, C{executeCommand} needs a way to discard all
855 output written to C{stderr}, as a means of simulating the shell
856 C{2>/dev/null} construct.
857 """
858 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
859 stderr = STDOUT
860 if ignoreStderr:
861 devnull = nullDevice()
862 stderr = os.open(devnull, os.O_RDWR)
863 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
864
871
872 """
873 Class holding runtime diagnostic information.
874
875 Diagnostic information is information that is useful to get from users for
876 debugging purposes. I'm consolidating it all here into one object.
877
878 @sort: __init__, __repr__, __str__
879 """
880
881
883 """
884 Constructor for the C{Diagnostics} class.
885 """
886
888 """
889 Official string representation for class instance.
890 """
891 return "Diagnostics()"
892
894 """
895 Informal string representation for class instance.
896 """
897 return self.__repr__()
898
900 """
901 Get a map containing all of the diagnostic values.
902 @return: Map from diagnostic name to diagnostic value.
903 """
904 values = {}
905 values['version'] = self.version
906 values['interpreter'] = self.interpreter
907 values['platform'] = self.platform
908 values['encoding'] = self.encoding
909 values['locale'] = self.locale
910 values['timestamp'] = self.timestamp
911 values['supported'] = self.supported
912 return values
913
915 """
916 Pretty-print diagnostic information to a file descriptor.
917 @param fd: File descriptor used to print information.
918 @param prefix: Prefix string (if any) to place onto printed lines
919 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
920 """
921 lines = self._buildDiagnosticLines(prefix)
922 for line in lines:
923 fd.write("%s\n" % line)
924
926 """
927 Pretty-print diagnostic information using a logger method.
928 @param method: Logger method to use for logging (i.e. logger.info)
929 @param prefix: Prefix string (if any) to place onto printed lines
930 """
931 lines = self._buildDiagnosticLines(prefix)
932 for line in lines:
933 method("%s" % line)
934
936 """
937 Build a set of pretty-printed diagnostic lines.
938 @param prefix: Prefix string (if any) to place onto printed lines
939 @return: List of strings, not terminated by newlines.
940 """
941 values = self.getValues()
942 keys = values.keys()
943 keys.sort()
944 tmax = Diagnostics._getMaxLength(keys) + 3
945 lines = []
946 for key in keys:
947 title = key.title()
948 title += (tmax - len(title)) * '.'
949 value = values[key]
950 line = "%s%s: %s" % (prefix, title, value)
951 lines.append(line)
952 return lines
953
954 @staticmethod
956 """
957 Get the maximum length from among a list of strings.
958 """
959 tmax = 0
960 for value in values:
961 if len(value) > tmax:
962 tmax = len(value)
963 return tmax
964
966 """
967 Property target to get the Cedar Backup version.
968 """
969 return "Cedar Backup %s (%s)" % (VERSION, DATE)
970
972 """
973 Property target to get the Python interpreter version.
974 """
975 version = sys.version_info
976 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
977
979 """
980 Property target to get the filesystem encoding.
981 """
982 return sys.getfilesystemencoding() or sys.getdefaultencoding()
983
1004
1006 """
1007 Property target to get the default locale that is in effect.
1008 """
1009 try:
1010 import locale
1011 return locale.getdefaultlocale()[0]
1012 except:
1013 return "(unknown)"
1014
1016 """
1017 Property target to get a current date/time stamp.
1018 """
1019 try:
1020 import datetime
1021 return datetime.datetime.utcnow().ctime() + " UTC"
1022 except:
1023 return "(unknown)"
1024
1026 """
1027 Property target to get the supported value.
1028 """
1029 return "Unsupported as of 11 Nov 2017"
1030
1031 version = property(_getVersion, None, None, "Cedar Backup version.")
1032 interpreter = property(_getInterpreter, None, None, "Python interpreter version.")
1033 platform = property(_getPlatform, None, None, "Platform identifying information.")
1034 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.")
1035 locale = property(_getLocale, None, None, "Locale that is in effect.")
1036 timestamp = property(_getTimestamp, None, None, "Current timestamp.")
1037 supported = property(_getSupported, None, None, "Whether the version is supported.")
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048 -def sortDict(d):
1049 """
1050 Returns the keys of the dictionary sorted by value.
1051
1052 There are cuter ways to do this in Python 2.4, but we were originally
1053 attempting to stay compatible with Python 2.3.
1054
1055 @param d: Dictionary to operate on
1056 @return: List of dictionary keys sorted in order by dictionary value.
1057 """
1058 items = d.items()
1059 items.sort(lambda x, y: cmp(x[1], y[1]))
1060 return [key for key, value in items]
1061
1062
1063
1064
1065
1066
1067 -def removeKeys(d, keys):
1068 """
1069 Removes all of the keys from the dictionary.
1070 The dictionary is altered in-place.
1071 Each key must exist in the dictionary.
1072 @param d: Dictionary to operate on
1073 @param keys: List of keys to remove
1074 @raise KeyError: If one of the keys does not exist
1075 """
1076 for key in keys:
1077 del d[key]
1078
1079
1080
1081
1082
1083
1084 -def convertSize(size, fromUnit, toUnit):
1085 """
1086 Converts a size in one unit to a size in another unit.
1087
1088 This is just a convenience function so that the functionality can be
1089 implemented in just one place. Internally, we convert values to bytes and
1090 then to the final unit.
1091
1092 The available units are:
1093
1094 - C{UNIT_BYTES} - Bytes
1095 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B
1096 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB
1097 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB
1098 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B
1099
1100 @param size: Size to convert
1101 @type size: Integer or float value in units of C{fromUnit}
1102
1103 @param fromUnit: Unit to convert from
1104 @type fromUnit: One of the units listed above
1105
1106 @param toUnit: Unit to convert to
1107 @type toUnit: One of the units listed above
1108
1109 @return: Number converted to new unit, as a float.
1110 @raise ValueError: If one of the units is invalid.
1111 """
1112 if size is None:
1113 raise ValueError("Cannot convert size of None.")
1114 if fromUnit == UNIT_BYTES:
1115 byteSize = float(size)
1116 elif fromUnit == UNIT_KBYTES:
1117 byteSize = float(size) * BYTES_PER_KBYTE
1118 elif fromUnit == UNIT_MBYTES:
1119 byteSize = float(size) * BYTES_PER_MBYTE
1120 elif fromUnit == UNIT_GBYTES:
1121 byteSize = float(size) * BYTES_PER_GBYTE
1122 elif fromUnit == UNIT_SECTORS:
1123 byteSize = float(size) * BYTES_PER_SECTOR
1124 else:
1125 raise ValueError("Unknown 'from' unit %s." % fromUnit)
1126 if toUnit == UNIT_BYTES:
1127 return byteSize
1128 elif toUnit == UNIT_KBYTES:
1129 return byteSize / BYTES_PER_KBYTE
1130 elif toUnit == UNIT_MBYTES:
1131 return byteSize / BYTES_PER_MBYTE
1132 elif toUnit == UNIT_GBYTES:
1133 return byteSize / BYTES_PER_GBYTE
1134 elif toUnit == UNIT_SECTORS:
1135 return byteSize / BYTES_PER_SECTOR
1136 else:
1137 raise ValueError("Unknown 'to' unit %s." % toUnit)
1138
1139
1140
1141
1142
1143
1144 -def displayBytes(bytes, digits=2):
1145 """
1146 Format a byte quantity so it can be sensibly displayed.
1147
1148 It's rather difficult to look at a number like "72372224 bytes" and get any
1149 meaningful information out of it. It would be more useful to see something
1150 like "69.02 MB". That's what this function does. Any time you want to display
1151 a byte value, i.e.::
1152
1153 print "Size: %s bytes" % bytes
1154
1155 Call this function instead::
1156
1157 print "Size: %s" % displayBytes(bytes)
1158
1159 What comes out will be sensibly formatted. The indicated number of digits
1160 will be listed after the decimal point, rounded based on whatever rules are
1161 used by Python's standard C{%f} string format specifier. (Values less than 1
1162 kB will be listed in bytes and will not have a decimal point, since the
1163 concept of a fractional byte is nonsensical.)
1164
1165 @param bytes: Byte quantity.
1166 @type bytes: Integer number of bytes.
1167
1168 @param digits: Number of digits to display after the decimal point.
1169 @type digits: Integer value, typically 2-5.
1170
1171 @return: String, formatted for sensible display.
1172 """
1173 if bytes is None:
1174 raise ValueError("Cannot display byte value of None.")
1175 bytes = float(bytes)
1176 if math.fabs(bytes) < BYTES_PER_KBYTE:
1177 fmt = "%.0f bytes"
1178 value = bytes
1179 elif math.fabs(bytes) < BYTES_PER_MBYTE:
1180 fmt = "%." + "%d" % digits + "f kB"
1181 value = bytes / BYTES_PER_KBYTE
1182 elif math.fabs(bytes) < BYTES_PER_GBYTE:
1183 fmt = "%." + "%d" % digits + "f MB"
1184 value = bytes / BYTES_PER_MBYTE
1185 else:
1186 fmt = "%." + "%d" % digits + "f GB"
1187 value = bytes / BYTES_PER_GBYTE
1188 return fmt % value
1189
1196 """
1197 Gets a reference to a named function.
1198
1199 This does some hokey-pokey to get back a reference to a dynamically named
1200 function. For instance, say you wanted to get a reference to the
1201 C{os.path.isdir} function. You could use::
1202
1203 myfunc = getFunctionReference("os.path", "isdir")
1204
1205 Although we won't bomb out directly, behavior is pretty much undefined if
1206 you pass in C{None} or C{""} for either C{module} or C{function}.
1207
1208 The only validation we enforce is that whatever we get back must be
1209 callable.
1210
1211 I derived this code based on the internals of the Python unittest
1212 implementation. I don't claim to completely understand how it works.
1213
1214 @param module: Name of module associated with function.
1215 @type module: Something like "os.path" or "CedarBackup2.util"
1216
1217 @param function: Name of function
1218 @type function: Something like "isdir" or "getUidGid"
1219
1220 @return: Reference to function associated with name.
1221
1222 @raise ImportError: If the function cannot be found.
1223 @raise ValueError: If the resulting reference is not callable.
1224
1225 @copyright: Some of this code, prior to customization, was originally part
1226 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python
1227 Software Foundation; All Rights Reserved.
1228 """
1229 parts = []
1230 if module is not None and module != "":
1231 parts = module.split(".")
1232 if function is not None and function != "":
1233 parts.append(function)
1234 copy = parts[:]
1235 while copy:
1236 try:
1237 module = __import__(string.join(copy, "."))
1238 break
1239 except ImportError:
1240 del copy[-1]
1241 if not copy: raise
1242 parts = parts[1:]
1243 obj = module
1244 for part in parts:
1245 obj = getattr(obj, part)
1246 if not callable(obj):
1247 raise ValueError("Reference to %s.%s is not callable." % (module, function))
1248 return obj
1249
1250
1251
1252
1253
1254
1255 -def getUidGid(user, group):
1256 """
1257 Get the uid/gid associated with a user/group pair
1258
1259 This is a no-op if user/group functionality is not available on the platform.
1260
1261 @param user: User name
1262 @type user: User name as a string
1263
1264 @param group: Group name
1265 @type group: Group name as a string
1266
1267 @return: Tuple C{(uid, gid)} matching passed-in user and group.
1268 @raise ValueError: If the ownership user/group values are invalid
1269 """
1270 if _UID_GID_AVAILABLE:
1271 try:
1272 uid = pwd.getpwnam(user)[2]
1273 gid = grp.getgrnam(group)[2]
1274 return (uid, gid)
1275 except Exception, e:
1276 logger.debug("Error looking up uid and gid for [%s:%s]: %s", user, group, e)
1277 raise ValueError("Unable to lookup up uid and gid for passed in user/group.")
1278 else:
1279 return (0, 0)
1280
1287 """
1288 Changes ownership of path to match the user and group.
1289
1290 This is a no-op if user/group functionality is not available on the
1291 platform, or if the either passed-in user or group is C{None}. Further, we
1292 won't even try to do it unless running as root, since it's unlikely to work.
1293
1294 @param path: Path whose ownership to change.
1295 @param user: User which owns file.
1296 @param group: Group which owns file.
1297 """
1298 if _UID_GID_AVAILABLE:
1299 if user is None or group is None:
1300 logger.debug("User or group is None, so not attempting to change owner on [%s].", path)
1301 elif not isRunningAsRoot():
1302 logger.debug("Not root, so not attempting to change owner on [%s].", path)
1303 else:
1304 try:
1305 (uid, gid) = getUidGid(user, group)
1306 os.chown(path, uid, gid)
1307 except Exception, e:
1308 logger.error("Error changing ownership of [%s]: %s", path, e)
1309
1316 """
1317 Indicates whether the program is running as the root user.
1318 """
1319 return os.getuid() == 0
1320
1327 """
1328 Splits a command line string into a list of arguments.
1329
1330 Unfortunately, there is no "standard" way to parse a command line string,
1331 and it's actually not an easy problem to solve portably (essentially, we
1332 have to emulate the shell argument-processing logic). This code only
1333 respects double quotes (C{"}) for grouping arguments, not single quotes
1334 (C{'}). Make sure you take this into account when building your command
1335 line.
1336
1337 Incidentally, I found this particular parsing method while digging around in
1338 Google Groups, and I tweaked it for my own use.
1339
1340 @param commandLine: Command line string
1341 @type commandLine: String, i.e. "cback --verbose stage store"
1342
1343 @return: List of arguments, suitable for passing to C{popen2}.
1344
1345 @raise ValueError: If the command line is None.
1346 """
1347 if commandLine is None:
1348 raise ValueError("Cannot split command line of None.")
1349 fields = re.findall('[^ "]+|"[^"]+"', commandLine)
1350 fields = [field.replace('"', '') for field in fields]
1351 return fields
1352
1359 """
1360 Resolves the real path to a command through the path resolver mechanism.
1361
1362 Both extensions and standard Cedar Backup functionality need a way to
1363 resolve the "real" location of various executables. Normally, they assume
1364 that these executables are on the system path, but some callers need to
1365 specify an alternate location.
1366
1367 Ideally, we want to handle this configuration in a central location. The
1368 Cedar Backup path resolver mechanism (a singleton called
1369 L{PathResolverSingleton}) provides the central location to store the
1370 mappings. This function wraps access to the singleton, and is what all
1371 functions (extensions or standard functionality) should call if they need to
1372 find a command.
1373
1374 The passed-in command must actually be a list, in the standard form used by
1375 all existing Cedar Backup code (something like C{["svnlook", ]}). The
1376 lookup will actually be done on the first element in the list, and the
1377 returned command will always be in list form as well.
1378
1379 If the passed-in command can't be resolved or no mapping exists, then the
1380 command itself will be returned unchanged. This way, we neatly fall back on
1381 default behavior if we have no sensible alternative.
1382
1383 @param command: Command to resolve.
1384 @type command: List form of command, i.e. C{["svnlook", ]}.
1385
1386 @return: Path to command or just command itself if no mapping exists.
1387 """
1388 singleton = PathResolverSingleton.getInstance()
1389 name = command[0]
1390 result = command[:]
1391 result[0] = singleton.lookup(name, name)
1392 return result
1393
1394
1395
1396
1397
1398
1399 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1400 """
1401 Executes a shell command, hopefully in a safe way.
1402
1403 This function exists to replace direct calls to C{os.popen} in the Cedar
1404 Backup code. It's not safe to call a function such as C{os.popen()} with
1405 untrusted arguments, since that can cause problems if the string contains
1406 non-safe variables or other constructs (imagine that the argument is
1407 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/;
1408 echo"} in the current environment).
1409
1410 Instead, it's safer to pass a list of arguments in the style supported bt
1411 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe}
1412 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}.
1413
1414 Under the normal case, this function will return a tuple of C{(status,
1415 None)} where the status is the wait-encoded return status of the call per
1416 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as
1417 C{True}, the function will return a tuple of C{(status, output)} where
1418 C{output} is a list of strings, one entry per line in the output from the
1419 command. Output is always logged to the C{outputLogger.info()} target,
1420 regardless of whether it's returned.
1421
1422 By default, C{stdout} and C{stderr} will be intermingled in the output.
1423 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be
1424 included in the output.
1425
1426 The C{doNotLog} parameter exists so that callers can force the function to
1427 not log command output to the debug log. Normally, you would want to log.
1428 However, if you're using this function to write huge output files (i.e.
1429 database backups written to C{stdout}) then you might want to avoid putting
1430 all that information into the debug log.
1431
1432 The C{outputFile} parameter exists to make it easier for a caller to push
1433 output into a file, i.e. as a substitute for redirection to a file. If this
1434 value is passed in, each time a line of output is generated, it will be
1435 written to the file using C{outputFile.write()}. At the end, the file
1436 descriptor will be flushed using C{outputFile.flush()}. The caller
1437 maintains responsibility for closing the file object appropriately.
1438
1439 @note: I know that it's a bit confusing that the command and the arguments
1440 are both lists. I could have just required the caller to pass in one big
1441 list. However, I think it makes some sense to keep the command (the
1442 constant part of what we're executing, i.e. C{"scp -B"}) separate from its
1443 arguments, even if they both end up looking kind of similar.
1444
1445 @note: You cannot redirect output via shell constructs (i.e. C{>file},
1446 C{2>/dev/null}, etc.) using this function. The redirection string would be
1447 passed to the command just like any other argument. However, you can
1448 implement the equivalent to redirection using C{ignoreStderr} and
1449 C{outputFile}, as discussed above.
1450
1451 @note: The operating system environment is partially sanitized before
1452 the command is invoked. See L{sanitizeEnvironment} for details.
1453
1454 @param command: Shell command to execute
1455 @type command: List of individual arguments that make up the command
1456
1457 @param args: List of arguments to the command
1458 @type args: List of additional arguments to the command
1459
1460 @param returnOutput: Indicates whether to return the output of the command
1461 @type returnOutput: Boolean C{True} or C{False}
1462
1463 @param ignoreStderr: Whether stderr should be discarded
1464 @type ignoreStderr: Boolean True or False
1465
1466 @param doNotLog: Indicates that output should not be logged.
1467 @type doNotLog: Boolean C{True} or C{False}
1468
1469 @param outputFile: File object that all output should be written to.
1470 @type outputFile: File object as returned from C{open()} or C{file()}.
1471
1472 @return: Tuple of C{(result, output)} as described above.
1473 """
1474 logger.debug("Executing command %s with args %s.", command, args)
1475 outputLogger.info("Executing command %s with args %s.", command, args)
1476 if doNotLog:
1477 logger.debug("Note: output will not be logged, per the doNotLog flag.")
1478 outputLogger.info("Note: output will not be logged, per the doNotLog flag.")
1479 output = []
1480 fields = command[:]
1481 fields.extend(args)
1482 try:
1483 sanitizeEnvironment()
1484 try:
1485 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1486 except OSError:
1487
1488
1489
1490 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1491 while True:
1492 line = pipe.stdout.readline()
1493 if not line: break
1494 if returnOutput: output.append(line)
1495 if outputFile is not None: outputFile.write(line)
1496 if not doNotLog: outputLogger.info(line[:-1])
1497 if outputFile is not None:
1498 try:
1499 outputFile.flush()
1500 except: pass
1501 if returnOutput:
1502 return (pipe.wait(), output)
1503 else:
1504 return (pipe.wait(), None)
1505 except OSError, e:
1506 try:
1507 if returnOutput:
1508 if output != []:
1509 return (pipe.wait(), output)
1510 else:
1511 return (pipe.wait(), [ e, ])
1512 else:
1513 return (pipe.wait(), None)
1514 except UnboundLocalError:
1515 if returnOutput:
1516 return (256, [])
1517 else:
1518 return (256, None)
1519
1526 """
1527 Calculates the age (in days) of a file.
1528
1529 The "age" of a file is the amount of time since the file was last used, per
1530 the most recent of the file's C{st_atime} and C{st_mtime} values.
1531
1532 Technically, we only intend this function to work with files, but it will
1533 probably work with anything on the filesystem.
1534
1535 @param path: Path to a file on disk.
1536
1537 @return: Age of the file in days (possibly fractional).
1538 @raise OSError: If the file doesn't exist.
1539 """
1540 currentTime = int(time.time())
1541 fileStats = os.stat(path)
1542 lastUse = max(fileStats.st_atime, fileStats.st_mtime)
1543 ageInSeconds = currentTime - lastUse
1544 ageInDays = ageInSeconds / SECONDS_PER_DAY
1545 return ageInDays
1546
1547
1548
1549
1550
1551
1552 -def mount(devicePath, mountPoint, fsType):
1553 """
1554 Mounts the indicated device at the indicated mount point.
1555
1556 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount
1557 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any
1558 filesystem type that is supported by C{mount} on your platform. If the type
1559 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may
1560 not work on all systems.
1561
1562 @note: This only works on platforms that have a concept of "mounting" a
1563 filesystem through a command-line C{"mount"} command, like UNIXes. It
1564 won't work on Windows.
1565
1566 @param devicePath: Path of device to be mounted.
1567 @param mountPoint: Path that device should be mounted at.
1568 @param fsType: Type of the filesystem assumed to be available via the device.
1569
1570 @raise IOError: If the device cannot be mounted.
1571 """
1572 if fsType is None:
1573 args = [ devicePath, mountPoint ]
1574 else:
1575 args = [ "-t", fsType, devicePath, mountPoint ]
1576 command = resolveCommand(MOUNT_COMMAND)
1577 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0]
1578 if result != 0:
1579 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1580
1581
1582
1583
1584
1585
1586 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1587 """
1588 Unmounts whatever device is mounted at the indicated mount point.
1589
1590 Sometimes, it might not be possible to unmount the mount point immediately,
1591 if there are still files open there. Use the C{attempts} and C{waitSeconds}
1592 arguments to indicate how many unmount attempts to make and how many seconds
1593 to wait between attempts. If you pass in zero attempts, no attempts will be
1594 made (duh).
1595
1596 If the indicated mount point is not really a mount point per
1597 C{os.path.ismount()}, then it will be ignored. This seems to be a safer
1598 check then looking through C{/etc/mtab}, since C{ismount()} is already in
1599 the Python standard library and is documented as working on all POSIX
1600 systems.
1601
1602 If C{removeAfter} is C{True}, then the mount point will be removed using
1603 C{os.rmdir()} after the unmount action succeeds. If for some reason the
1604 mount point is not a directory, then it will not be removed.
1605
1606 @note: This only works on platforms that have a concept of "mounting" a
1607 filesystem through a command-line C{"mount"} command, like UNIXes. It
1608 won't work on Windows.
1609
1610 @param mountPoint: Mount point to be unmounted.
1611 @param removeAfter: Remove the mount point after unmounting it.
1612 @param attempts: Number of times to attempt the unmount.
1613 @param waitSeconds: Number of seconds to wait between repeated attempts.
1614
1615 @raise IOError: If the mount point is still mounted after attempts are exhausted.
1616 """
1617 if os.path.ismount(mountPoint):
1618 for attempt in range(0, attempts):
1619 logger.debug("Making attempt %d to unmount [%s].", attempt, mountPoint)
1620 command = resolveCommand(UMOUNT_COMMAND)
1621 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0]
1622 if result != 0:
1623 logger.error("Error [%d] unmounting [%s] on attempt %d.", result, mountPoint, attempt)
1624 elif os.path.ismount(mountPoint):
1625 logger.error("After attempt %d, [%s] is still mounted.", attempt, mountPoint)
1626 else:
1627 logger.debug("Successfully unmounted [%s] on attempt %d.", mountPoint, attempt)
1628 break
1629 if attempt+1 < attempts:
1630 if waitSeconds > 0:
1631 logger.info("Sleeping %d second(s) before next unmount attempt.", waitSeconds)
1632 time.sleep(waitSeconds)
1633 else:
1634 if os.path.ismount(mountPoint):
1635 raise IOError("Unable to unmount [%s] after %d attempts." % (mountPoint, attempts))
1636 logger.info("Mount point [%s] seems to have finally gone away.", mountPoint)
1637 if os.path.isdir(mountPoint) and removeAfter:
1638 logger.debug("Removing mount point [%s].", mountPoint)
1639 os.rmdir(mountPoint)
1640
1647 """
1648 Indicates whether a specific filesystem device is currently mounted.
1649
1650 We determine whether the device is mounted by looking through the system's
1651 C{mtab} file. This file shows every currently-mounted filesystem, ordered
1652 by device. We only do the check if the C{mtab} file exists and is readable.
1653 Otherwise, we assume that the device is not mounted.
1654
1655 @note: This only works on platforms that have a concept of an mtab file
1656 to show mounted volumes, like UNIXes. It won't work on Windows.
1657
1658 @param devicePath: Path of device to be checked
1659
1660 @return: True if device is mounted, false otherwise.
1661 """
1662 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK):
1663 realPath = os.path.realpath(devicePath)
1664 lines = open(MTAB_FILE).readlines()
1665 for line in lines:
1666 (mountDevice, mountPoint, remainder) = line.split(None, 2)
1667 if mountDevice in [ devicePath, realPath, ]:
1668 logger.debug("Device [%s] is mounted at [%s].", devicePath, mountPoint)
1669 return True
1670 return False
1671
1678
1679 r"""
1680 Safely encodes a filesystem path.
1681
1682 Many Python filesystem functions, such as C{os.listdir}, behave differently
1683 if they are passed unicode arguments versus simple string arguments. For
1684 instance, C{os.listdir} generally returns unicode path names if it is passed
1685 a unicode argument, and string pathnames if it is passed a string argument.
1686
1687 However, this behavior often isn't as consistent as we might like. As an example,
1688 C{os.listdir} "gives up" if it finds a filename that it can't properly encode
1689 given the current locale settings. This means that the returned list is
1690 a mixed set of unicode and simple string paths. This has consequences later,
1691 because other filesystem functions like C{os.path.join} will blow up if they
1692 are given one string path and one unicode path.
1693
1694 On comp.lang.python, Martin v. Löwis explained the C{os.listdir} behavior
1695 like this::
1696
1697 The operating system (POSIX) does not have the inherent notion that file
1698 names are character strings. Instead, in POSIX, file names are primarily
1699 byte strings. There are some bytes which are interpreted as characters
1700 (e.g. '\x2e', which is '.', or '\x2f', which is '/'), but apart from
1701 that, most OS layers think these are just bytes.
1702
1703 Now, most *people* think that file names are character strings. To
1704 interpret a file name as a character string, you need to know what the
1705 encoding is to interpret the file names (which are byte strings) as
1706 character strings.
1707
1708 There is, unfortunately, no operating system API to carry the notion of a
1709 file system encoding. By convention, the locale settings should be used
1710 to establish this encoding, in particular the LC_CTYPE facet of the
1711 locale. This is defined in the environment variables LC_CTYPE, LC_ALL,
1712 and LANG (searched in this order).
1713
1714 If LANG is not set, the "C" locale is assumed, which uses ASCII as its
1715 file system encoding. In this locale, '\xe2\x99\xaa\xe2\x99\xac' is not a
1716 valid file name (at least it cannot be interpreted as characters, and
1717 hence not be converted to Unicode).
1718
1719 Now, your Python script has requested that all file names *should* be
1720 returned as character (ie. Unicode) strings, but Python cannot comply,
1721 since there is no way to find out what this byte string means, in terms
1722 of characters.
1723
1724 So we have three options:
1725
1726 1. Skip this string, only return the ones that can be converted to Unicode.
1727 Give the user the impression the file does not exist.
1728 2. Return the string as a byte string
1729 3. Refuse to listdir altogether, raising an exception (i.e. return nothing)
1730
1731 Python has chosen alternative 2, allowing the application to implement 1
1732 or 3 on top of that if it wants to (or come up with other strategies,
1733 such as user feedback).
1734
1735 As a solution, he suggests that rather than passing unicode paths into the
1736 filesystem functions, that I should sensibly encode the path first. That is
1737 what this function accomplishes. Any function which takes a filesystem path
1738 as an argument should encode it first, before using it for any other purpose.
1739
1740 I confess I still don't completely understand how this works. On a system
1741 with filesystem encoding "ISO-8859-1", a path C{u"\xe2\x99\xaa\xe2\x99\xac"}
1742 is converted into the string C{"\xe2\x99\xaa\xe2\x99\xac"}. However, on a
1743 system with a "utf-8" encoding, the result is a completely different string:
1744 C{"\xc3\xa2\xc2\x99\xc2\xaa\xc3\xa2\xc2\x99\xc2\xac"}. A quick test where I
1745 write to the first filename and open the second proves that the two strings
1746 represent the same file on disk, which is all I really care about.
1747
1748 @note: As a special case, if C{path} is C{None}, then this function will
1749 return C{None}.
1750
1751 @note: To provide several examples of encoding values, my Debian sarge box
1752 with an ext3 filesystem has Python filesystem encoding C{ISO-8859-1}. User
1753 Anarcat's Debian box with a xfs filesystem has filesystem encoding
1754 C{ANSI_X3.4-1968}. Both my iBook G4 running Mac OS X 10.4 and user Dag
1755 Rende's SuSE 9.3 box both have filesystem encoding C{UTF-8}.
1756
1757 @note: Just because a filesystem has C{UTF-8} encoding doesn't mean that it
1758 will be able to handle all extended-character filenames. For instance,
1759 certain extended-character (but not UTF-8) filenames -- like the ones in the
1760 regression test tar file C{test/data/tree13.tar.gz} -- are not valid under
1761 Mac OS X, and it's not even possible to extract them from the tarfile on
1762 that platform.
1763
1764 @param path: Path to encode
1765
1766 @return: Path, as a string, encoded appropriately
1767 @raise ValueError: If the path cannot be encoded properly.
1768 """
1769 if path is None:
1770 return path
1771 try:
1772 if isinstance(path, unicode):
1773 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
1774 path = path.encode(encoding)
1775 return path
1776 except UnicodeError:
1777 raise ValueError("Path could not be safely encoded as %s." % encoding)
1778
1785 """
1786 Attempts to portably return the null device on this system.
1787
1788 The null device is something like C{/dev/null} on a UNIX system. The name
1789 varies on other platforms.
1790 """
1791 return os.devnull
1792
1799 """
1800 Converts English day name to numeric day of week as from C{time.localtime}.
1801
1802 For instance, the day C{monday} would be converted to the number C{0}.
1803
1804 @param dayName: Day of week to convert
1805 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1806
1807 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible.
1808 """
1809 if dayName.lower() == "monday":
1810 return 0
1811 elif dayName.lower() == "tuesday":
1812 return 1
1813 elif dayName.lower() == "wednesday":
1814 return 2
1815 elif dayName.lower() == "thursday":
1816 return 3
1817 elif dayName.lower() == "friday":
1818 return 4
1819 elif dayName.lower() == "saturday":
1820 return 5
1821 elif dayName.lower() == "sunday":
1822 return 6
1823 else:
1824 return -1
1825
1832 """
1833 Indicates whether "today" is the backup starting day per configuration.
1834
1835 If the current day's English name matches the indicated starting day, then
1836 today is a starting day.
1837
1838 @param startingDay: Configured starting day.
1839 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1840
1841 @return: Boolean indicating whether today is the starting day.
1842 """
1843 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay)
1844 if value:
1845 logger.debug("Today is the start of the week.")
1846 else:
1847 logger.debug("Today is NOT the start of the week.")
1848 return value
1849
1856 """
1857 Returns a "normalized" path based on a path name.
1858
1859 A normalized path is a representation of a path that is also a valid file
1860 name. To make a valid file name out of a complete path, we have to convert
1861 or remove some characters that are significant to the filesystem -- in
1862 particular, the path separator and any leading C{'.'} character (which would
1863 cause the file to be hidden in a file listing).
1864
1865 Note that this is a one-way transformation -- you can't safely derive the
1866 original path from the normalized path.
1867
1868 To normalize a path, we begin by looking at the first character. If the
1869 first character is C{'/'} or C{'\\'}, it gets removed. If the first
1870 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the
1871 rest of the path and convert all remaining C{'/'} or C{'\\'} characters
1872 C{'-'}, and all remaining whitespace characters to C{'_'}.
1873
1874 As a special case, a path consisting only of a single C{'/'} or C{'\\'}
1875 character will be converted to C{'-'}.
1876
1877 @param path: Path to normalize
1878
1879 @return: Normalized path as described above.
1880
1881 @raise ValueError: If the path is None
1882 """
1883 if path is None:
1884 raise ValueError("Cannot normalize path None.")
1885 elif len(path) == 0:
1886 return path
1887 elif path == "/" or path == "\\":
1888 return "-"
1889 else:
1890 normalized = path
1891 normalized = re.sub(r"^\/", "", normalized)
1892 normalized = re.sub(r"^\\", "", normalized)
1893 normalized = re.sub(r"^\.", "_", normalized)
1894 normalized = re.sub(r"\/", "-", normalized)
1895 normalized = re.sub(r"\\", "-", normalized)
1896 normalized = re.sub(r"\s", "_", normalized)
1897 return normalized
1898
1905 """
1906 Sanitizes the operating system environment.
1907
1908 The operating system environment is contained in C{os.environ}. This method
1909 sanitizes the contents of that dictionary.
1910
1911 Currently, all it does is reset the locale (removing C{$LC_*}) and set the
1912 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count
1913 on consistent localization regardless of what the end-user has configured.
1914 This is important for code that needs to parse program output.
1915
1916 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already
1917 set to the proper value, it is not re-set, so we can avoid the memory leaks
1918 that are documented to occur on BSD-based systems.
1919
1920 @return: Copy of the sanitized environment.
1921 """
1922 for var in LOCALE_VARS:
1923 if os.environ.has_key(var):
1924 del os.environ[var]
1925 if os.environ.has_key(LANG_VAR):
1926 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE:
1927 os.environ[LANG_VAR] = DEFAULT_LANGUAGE
1928 return os.environ.copy()
1929
1936 """
1937 Deference a soft link, optionally normalizing it to an absolute path.
1938 @param path: Path of link to dereference
1939 @param absolute: Whether to normalize the result to an absolute path
1940 @return: Dereferenced path, or original path if original is not a link.
1941 """
1942 if os.path.islink(path):
1943 result = os.readlink(path)
1944 if absolute and not os.path.isabs(result):
1945 result = os.path.abspath(os.path.join(os.path.dirname(path), result))
1946 return result
1947 return path
1948
1949
1950
1951
1952
1953
1954 -def checkUnique(prefix, values):
1955 """
1956 Checks that all values are unique.
1957
1958 The values list is checked for duplicate values. If there are
1959 duplicates, an exception is thrown. All duplicate values are listed in
1960 the exception.
1961
1962 @param prefix: Prefix to use in the thrown exception
1963 @param values: List of values to check
1964
1965 @raise ValueError: If there are duplicates in the list
1966 """
1967 values.sort()
1968 duplicates = []
1969 for i in range(1, len(values)):
1970 if values[i-1] == values[i]:
1971 duplicates.append(values[i])
1972 if duplicates:
1973 raise ValueError("%s %s" % (prefix, duplicates))
1974
1981 """
1982 Parses a list of values out of a comma-separated string.
1983
1984 The items in the list are split by comma, and then have whitespace
1985 stripped. As a special case, if C{commaString} is C{None}, then C{None}
1986 will be returned.
1987
1988 @param commaString: List of values in comma-separated string format.
1989 @return: Values from commaString split into a list, or C{None}.
1990 """
1991 if commaString is None:
1992 return None
1993 else:
1994 pass1 = commaString.split(",")
1995 pass2 = []
1996 for item in pass1:
1997 item = item.strip()
1998 if len(item) > 0:
1999 pass2.append(item)
2000 return pass2
2001