diff --git a/Duplicati.sln b/Duplicati.sln index a2a077daf..dd6964eae 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -1,8 +1,6 @@  Microsoft Visual Studio Solution File, Format Version 9.00 # Visual Studio 2005 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati", "Duplicati\Duplicati.csproj", "{1470F75F-D364-4010-91D0-FD4DDA1C5159}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LightDatamodel", "..\LightDatamodel\LightDatamodel\LightDatamodel.csproj", "{CAE8433B-F013-474E-B7FC-4DE0D5933AD1}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Datamodel", "Duplicati\Datamodel\Datamodel.csproj", "{B60D0993-B0C5-457A-90D9-DB6EEB33CE47}" @@ -15,10 +13,6 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {1470F75F-D364-4010-91D0-FD4DDA1C5159}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1470F75F-D364-4010-91D0-FD4DDA1C5159}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1470F75F-D364-4010-91D0-FD4DDA1C5159}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1470F75F-D364-4010-91D0-FD4DDA1C5159}.Release|Any CPU.Build.0 = Release|Any CPU {CAE8433B-F013-474E-B7FC-4DE0D5933AD1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {CAE8433B-F013-474E-B7FC-4DE0D5933AD1}.Debug|Any CPU.Build.0 = Debug|Any CPU {CAE8433B-F013-474E-B7FC-4DE0D5933AD1}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/Update DEBUG Environment.bat b/Update DEBUG Environment.bat new file mode 100644 index 000000000..b87530a19 --- /dev/null +++ b/Update DEBUG Environment.bat @@ -0,0 +1,6 @@ +xcopy /Y /E thirdparty\gpg Duplicati\bin\debug\gpp\ +xcopy /Y /E thirdparty\python25 Duplicati\bin\debug\python25\ +xcopy /Y /E thirdparty\duplicity Duplicati\bin\debug\duplicity\ +xcopy /Y thirdparty\GnuPGInterface\GnuPGInterface.py Duplicati\bin\debug\duplicity\GnuPGInterface.py +xcopy /Y /E thirdparty\boto Duplicati\bin\debug\python25\lib\boto +pause \ No newline at end of file diff --git a/Update RELEASE Environment.bat b/Update RELEASE Environment.bat new file mode 100644 index 000000000..d585be3d0 --- /dev/null +++ b/Update RELEASE Environment.bat @@ -0,0 +1,6 @@ +xcopy /Y /E thirdparty\gpg Duplicati\bin\release\gpp\ +xcopy /Y /E thirdparty\python25 Duplicati\bin\release\python25\ +xcopy /Y /E thirdparty\duplicity Duplicati\bin\release\duplicity\ +xcopy /Y thirdparty\GnuPGInterface\GnuPGInterface.py Duplicati\bin\release\duplicity\GnuPGInterface.py +xcopy /Y /E thirdparty\boto Duplicati\bin\release\python25\lib\boto +pause \ No newline at end of file diff --git a/thirdparty/GnuPGInterface/GnuPGInterface.py b/thirdparty/GnuPGInterface/GnuPGInterface.py new file mode 100644 index 000000000..9d832599e --- /dev/null +++ b/thirdparty/GnuPGInterface/GnuPGInterface.py @@ -0,0 +1,729 @@ +"""Interface to GNU Privacy Guard (GnuPG) + +GnuPGInterface is a Python module to interface with GnuPG. +It concentrates on interacting with GnuPG via filehandles, +providing access to control GnuPG via versatile and extensible means. + +This module is based on GnuPG::Interface, a Perl module by the same author. + +Normally, using this module will involve creating a +GnuPG object, setting some options in it's 'options' data member +(which is of type Options), creating some pipes +to talk with GnuPG, and then calling the run() method, which will +connect those pipes to the GnuPG process. run() returns a +Process object, which contains the filehandles to talk to GnuPG with. + +Example code: + +>>> import GnuPGInterface +>>> +>>> plaintext = "Three blind mice" +>>> passphrase = "This is the passphrase" +>>> +>>> gnupg = GnuPGInterface.GnuPG() +>>> gnupg.options.armor = 1 +>>> gnupg.options.meta_interactive = 0 +>>> gnupg.options.extra_args.append('--no-secmem-warning') +>>> +>>> # Normally we might specify something in +>>> # gnupg.options.recipients, like +>>> # gnupg.options.recipients = [ '0xABCD1234', 'bob@foo.bar' ] +>>> # but since we're doing symmetric-only encryption, it's not needed. +>>> # If you are doing standard, public-key encryption, using +>>> # --encrypt, you will need to specify recipients before +>>> # calling gnupg.run() +>>> +>>> # First we'll encrypt the test_text input symmetrically +>>> p1 = gnupg.run(['--symmetric'], +... create_fhs=['stdin', 'stdout', 'passphrase']) +>>> +>>> p1.handles['passphrase'].write(passphrase) +>>> p1.handles['passphrase'].close() +>>> +>>> p1.handles['stdin'].write(plaintext) +>>> p1.handles['stdin'].close() +>>> +>>> ciphertext = p1.handles['stdout'].read() +>>> p1.handles['stdout'].close() +>>> +>>> # process cleanup +>>> p1.wait() +>>> +>>> # Now we'll decrypt what we just encrypted it, +>>> # using the convience method to get the +>>> # passphrase to GnuPG +>>> gnupg.passphrase = passphrase +>>> +>>> p2 = gnupg.run(['--decrypt'], create_fhs=['stdin', 'stdout']) +>>> +>>> p2.handles['stdin'].write(ciphertext) +>>> p2.handles['stdin'].close() +>>> +>>> decrypted_plaintext = p2.handles['stdout'].read() +>>> p2.handles['stdout'].close() +>>> +>>> # process cleanup +>>> p2.wait() +>>> +>>> # Our decrypted plaintext: +>>> decrypted_plaintext +'Three blind mice' +>>> +>>> # ...and see it's the same as what we orignally encrypted +>>> assert decrypted_plaintext == plaintext, \ + "GnuPG decrypted output does not match original input" +>>> +>>> +>>> ################################################## +>>> # Now let's trying using run()'s attach_fhs paramter +>>> +>>> # we're assuming we're running on a unix... +>>> input = open('/etc/motd') +>>> +>>> p1 = gnupg.run(['--symmetric'], create_fhs=['stdout'], +... attach_fhs={'stdin': input}) +>>> +>>> # GnuPG will read the stdin from /etc/motd +>>> ciphertext = p1.handles['stdout'].read() +>>> +>>> # process cleanup +>>> p1.wait() +>>> +>>> # Now let's run the output through GnuPG +>>> # We'll write the output to a temporary file, +>>> import tempfile +>>> temp = tempfile.TemporaryFile() +>>> +>>> p2 = gnupg.run(['--decrypt'], create_fhs=['stdin'], +... attach_fhs={'stdout': temp}) +>>> +>>> # give GnuPG our encrypted stuff from the first run +>>> p2.handles['stdin'].write(ciphertext) +>>> p2.handles['stdin'].close() +>>> +>>> # process cleanup +>>> p2.wait() +>>> +>>> # rewind the tempfile and see what GnuPG gave us +>>> temp.seek(0) +>>> decrypted_plaintext = temp.read() +>>> +>>> # compare what GnuPG decrypted with our original input +>>> input.seek(0) +>>> input_data = input.read() +>>> +>>> assert decrypted_plaintext == input_data, \ + "GnuPG decrypted output does not match original input" + +To do things like public-key encryption, simply pass do something +like: + +gnupg.passphrase = 'My passphrase' +gnupg.options.recipients = [ 'bob@foobar.com' ] +gnupg.run( ['--sign', '--encrypt'], create_fhs=..., attach_fhs=...) + +Here is an example of subclassing GnuPGInterface.GnuPG, +so that it has an encrypt_string() method that returns +ciphertext. + +>>> import GnuPGInterface +>>> +>>> class MyGnuPG(GnuPGInterface.GnuPG): +... +... def __init__(self): +... GnuPGInterface.GnuPG.__init__(self) +... self.setup_my_options() +... +... def setup_my_options(self): +... self.options.armor = 1 +... self.options.meta_interactive = 0 +... self.options.extra_args.append('--no-secmem-warning') +... +... def encrypt_string(self, string, recipients): +... gnupg.options.recipients = recipients # a list! +... +... proc = gnupg.run(['--encrypt'], create_fhs=['stdin', 'stdout']) +... +... proc.handles['stdin'].write(string) +... proc.handles['stdin'].close() +... +... output = proc.handles['stdout'].read() +... proc.handles['stdout'].close() +... +... proc.wait() +... return output +... +>>> gnupg = MyGnuPG() +>>> ciphertext = gnupg.encrypt_string("The secret", ['0x260C4FA3']) +>>> +>>> # just a small sanity test here for doctest +>>> import types +>>> assert isinstance(ciphertext, types.StringType), \ + "What GnuPG gave back is not a string!" + +Here is an example of generating a key: +>>> import GnuPGInterface +>>> gnupg = GnuPGInterface.GnuPG() +>>> gnupg.options.meta_interactive = 0 +>>> +>>> # We will be creative and use the logger filehandle to capture +>>> # what GnuPG says this time, instead stderr; no stdout to listen to, +>>> # but we capture logger to surpress the dry-run command. +>>> # We also have to capture stdout since otherwise doctest complains; +>>> # Normally you can let stdout through when generating a key. +>>> +>>> proc = gnupg.run(['--gen-key'], create_fhs=['stdin', 'stdout', +... 'logger']) +>>> +>>> proc.handles['stdin'].write('''Key-Type: DSA +... Key-Length: 1024 +... # We are only testing syntax this time, so dry-run +... %dry-run +... Subkey-Type: ELG-E +... Subkey-Length: 1024 +... Name-Real: Joe Tester +... Name-Comment: with stupid passphrase +... Name-Email: joe@foo.bar +... Expire-Date: 2y +... Passphrase: abc +... %pubring foo.pub +... %secring foo.sec +... ''') +>>> +>>> proc.handles['stdin'].close() +>>> +>>> report = proc.handles['logger'].read() +>>> proc.handles['logger'].close() +>>> +>>> proc.wait() + + +COPYRIGHT: + +Copyright (C) 2001 Frank J. Tobin, ftobin@neverending.org + +LICENSE: + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +or see http://www.gnu.org/copyleft/lesser.html +""" + +import os +import sys +import subprocess +# import fcntl + +__author__ = "Frank J. Tobin, ftobin@neverending.org" +__version__ = "0.3.2" +__revision__ = "$Id: GnuPGInterface.py,v 1.22 2002/01/11 20:22:04 ftobin Exp $" + +# "standard" filehandles attached to processes +_stds = [ 'stdin', 'stdout', 'stderr' ] + +# the permissions each type of fh needs to be opened with +_fd_modes = { 'stdin': 'w', + 'stdout': 'r', + 'stderr': 'r', + 'passphrase': 'w', + 'command': 'w', + 'logger': 'r', + 'status': 'r' + } + +# correlation between handle names and the arguments we'll pass +_fd_options = { 'passphrase': '--passphrase-fd', + 'logger': '--logger-fd', + 'status': '--status-fd', + 'command': '--command-fd' } + +class GnuPG: + """Class instances represent GnuPG. + + Instance attributes of a GnuPG object are: + + * call -- string to call GnuPG with. Defaults to "gpg" + + * passphrase -- Since it is a common operation + to pass in a passphrase to GnuPG, + and working with the passphrase filehandle mechanism directly + can be mundane, if set, the passphrase attribute + works in a special manner. If the passphrase attribute is set, + and no passphrase file object is sent in to run(), + then GnuPG instnace will take care of sending the passphrase to + GnuPG, the executable instead of having the user sent it in manually. + + * options -- Object of type GnuPGInterface.Options. + Attribute-setting in options determines + the command-line options used when calling GnuPG. + """ + + def __init__(self): + self.call = 'gpg' + self.passphrase = None + self.options = Options() + + def run(self, gnupg_commands, args=None, create_fhs=None, attach_fhs=None): + """Calls GnuPG with the list of string commands gnupg_commands, + complete with prefixing dashes. + For example, gnupg_commands could be + '["--sign", "--encrypt"]' + Returns a GnuPGInterface.Process object. + + args is an optional list of GnuPG command arguments (not options), + such as keyID's to export, filenames to process, etc. + + create_fhs is an optional list of GnuPG filehandle + names that will be set as keys of the returned Process object's + 'handles' attribute. The generated filehandles can be used + to communicate with GnuPG via standard input, standard output, + the status-fd, passphrase-fd, etc. + + Valid GnuPG filehandle names are: + * stdin + * stdout + * stderr + * status + * passphase + * command + * logger + + The purpose of each filehandle is described in the GnuPG + documentation. + + attach_fhs is an optional dictionary with GnuPG filehandle + names mapping to opened files. GnuPG will read or write + to the file accordingly. For example, if 'my_file' is an + opened file and 'attach_fhs[stdin] is my_file', then GnuPG + will read its standard input from my_file. This is useful + if you want GnuPG to read/write to/from an existing file. + For instance: + + f = open("encrypted.gpg") + gnupg.run(["--decrypt"], attach_fhs={'stdin': f}) + + Using attach_fhs also helps avoid system buffering + issues that can arise when using create_fhs, which + can cause the process to deadlock. + + If not mentioned in create_fhs or attach_fhs, + GnuPG filehandles which are a std* (stdin, stdout, stderr) + are defaulted to the running process' version of handle. + Otherwise, that type of handle is simply not used when calling GnuPG. + For example, if you do not care about getting data from GnuPG's + status filehandle, simply do not specify it. + + run() returns a Process() object which has a 'handles' + which is a dictionary mapping from the handle name + (such as 'stdin' or 'stdout') to the respective + newly-created FileObject connected to the running GnuPG process. + For instance, if the call was + + process = gnupg.run(["--decrypt"], stdin=1) + + after run returns 'process.handles["stdin"]' + is a FileObject connected to GnuPG's standard input, + and can be written to. + """ + + if args == None: args = [] + if create_fhs == None: create_fhs = [] + if attach_fhs == None: attach_fhs = {} + + for std in _stds: + if not attach_fhs.has_key(std) \ + and std not in create_fhs: + attach_fhs.setdefault(std, getattr(sys, std)) + + handle_passphrase = 0 + + if self.passphrase != None \ + and not attach_fhs.has_key('passphrase') \ + and 'passphrase' not in create_fhs: + handle_passphrase = 1 + create_fhs.append('passphrase') + + process = self._attach_fork_exec(gnupg_commands, args, + create_fhs, attach_fhs) + + if handle_passphrase: + passphrase_fh = process.handles['passphrase'] + passphrase_fh.write( self.passphrase ) + passphrase_fh.close() + del process.handles['passphrase'] + + return process + + + def _attach_fork_exec(self, gnupg_commands, args, create_fhs, attach_fhs): + """This is like run(), but without the passphrase-helping + (note that run() calls this).""" + + process = Process() + + for fh_name in create_fhs + attach_fhs.keys(): + if not _fd_modes.has_key(fh_name): + raise KeyError, \ + "unrecognized filehandle name '%s'; must be one of %s" \ + % (fh_name, _fd_modes.keys()) + + for fh_name in create_fhs: + # make sure the user doesn't specify a filehandle + # to be created *and* attached + if attach_fhs.has_key(fh_name): + raise ValueError, \ + "cannot have filehandle '%s' in both create_fhs and attach_fhs" \ + % fh_name + + + + process_arg = self._as_child(process, gnupg_commands, args, attach_fhs) + + + if not attach_fhs.has_key("stdin"): + process.handles["stdin"] = process_arg.stdin + process.closehandles["stdin"] = process_arg.stdin + else: + process.handles["stdin"] = attach_fhs["stdin"] + + if not attach_fhs.has_key("stdout"): + process.handles["stdout"] = process_arg.stdout + process.closehandles["stdout"] = process_arg.stdout + else: + process.handles["stdout"] = attach_fhs["stdout"] + + if not attach_fhs.has_key("stderr"): + process.handles["stderr"] = process_arg.stderr + process.closehandles["stderr"] = process_arg.stderr + else: + process.handles["stderr"] = attach_fhs["stderr"] + + process.handles["passphrase"] = FakeFile(process.handles["stdin"]) + + process.args = process_arg + process.pid = process_arg.pid + return process + + + def _as_parent(self, process): + """Stuff run after forking in parent""" + for k, p in process._pipes.items(): + if not p.direct: + os.close(p.child) + process.handles[k] = os.fdopen(p.parent, _fd_modes[k]) + + # user doesn't need these + #del process._pipes + + return process + + + def _as_child(self, process, gnupg_commands, args, attach_fhs): + """Stuff run after forking in child""" + # child + + fd_args = [] + + k = 'passphrase' + fd_args.extend([ _fd_options[k], "%d" % 0 ]) + + command = [ self.call ] + fd_args + self.options.get_args() \ + + gnupg_commands + args + + stdin = subprocess.PIPE + stdout = subprocess.PIPE + stderr = subprocess.PIPE + + if attach_fhs.has_key("stdin"): + stdin = attach_fhs['stdin'] + + if attach_fhs.has_key("stdout"): + stdout = attach_fhs['stdout'] + + if attach_fhs.has_key("stderr"): + stderr = attach_fhs['stderr'] + + #os.execvp( command[0], command ) + #return subprocess.Popen(command, 0, None, 4, 3, 5) + #return subprocess.Popen(command, 0, None, stdin, stdout, stderr) + return subprocess.Popen(command, 0, None, stdin, stdout, stderr) + #return subprocess.Popen(command, 0, None, process._pipes['stdin'].child, process._pipes['stdout'].child, process._pipes['stderr'].child) + +class FakeFile: + """Simple wrapper for a file, where close is substituted with linefeed""" + + def __init__(self, file): + self._file = file + self._closed = 0 + + def write(self, arg): + if self._closed: + raise ValueError, \ + "handle is closed" + self._file.write(arg) + + def close(self): + self.write("\n") + self._closed = 1 + + +class Pipe: + """simple struct holding stuff about pipes we use""" + def __init__(self, parent, child, direct): + self.parent = parent + self.child = child + self.direct = direct + + +class Options: + """Objects of this class encompass options passed to GnuPG. + This class is responsible for determining command-line arguments + which are based on options. It can be said that a GnuPG + object has-a Options object in its options attribute. + + Attributes which correlate directly to GnuPG options: + + Each option here defaults to false or None, and is described in + GnuPG documentation. + + Booleans (set these attributes to booleans) + + * armor + * no_greeting + * no_verbose + * quiet + * batch + * always_trust + * rfc1991 + * openpgp + * force_v3_sigs + * no_options + * textmode + + Strings (set these attributes to strings) + + * homedir + * default_key + * comment + * compress_algo + * options + + Lists (set these attributes to lists) + + * recipients (***NOTE*** plural of 'recipient') + * encrypt_to + + Meta options + + Meta options are options provided by this module that do + not correlate directly to any GnuPG option by name, + but are rather bundle of options used to accomplish + a specific goal, such as obtaining compatibility with PGP 5. + The actual arguments each of these reflects may change with time. Each + defaults to false unless otherwise specified. + + meta_pgp_5_compatible -- If true, arguments are generated to try + to be compatible with PGP 5.x. + + meta_pgp_2_compatible -- If true, arguments are generated to try + to be compatible with PGP 2.x. + + meta_interactive -- If false, arguments are generated to try to + help the using program use GnuPG in a non-interactive + environment, such as CGI scripts. Default is true. + + extra_args -- Extra option arguments may be passed in + via the attribute extra_args, a list. + + >>> import GnuPGInterface + >>> + >>> gnupg = GnuPGInterface.GnuPG() + >>> gnupg.options.armor = 1 + >>> gnupg.options.recipients = ['Alice', 'Bob'] + >>> gnupg.options.extra_args = ['--no-secmem-warning'] + >>> + >>> # no need for users to call this normally; just for show here + >>> gnupg.options.get_args() + ['--armor', '--recipient', 'Alice', '--recipient', 'Bob', '--no-secmem-warning'] + """ + + def __init__(self): + # booleans + self.armor = 0 + self.no_greeting = 0 + self.verbose = 0 + self.no_verbose = 0 + self.quiet = 0 + self.batch = 0 + self.always_trust = 0 + self.rfc1991 = 0 + self.openpgp = 0 + self.force_v3_sigs = 0 + self.no_options = 0 + self.textmode = 0 + + # meta-option booleans + self.meta_pgp_5_compatible = 0 + self.meta_pgp_2_compatible = 0 + self.meta_interactive = 1 + + # strings + self.homedir = None + self.default_key = None + self.comment = None + self.compress_algo = None + self.options = None + + # lists + self.encrypt_to = [] + self.recipients = [] + + # miscellaneous arguments + self.extra_args = [] + + def get_args( self ): + """Generate a list of GnuPG arguments based upon attributes.""" + + return self.get_meta_args() + self.get_standard_args() + self.extra_args + + def get_standard_args( self ): + """Generate a list of standard, non-meta or extra arguments""" + args = [] + if self.homedir != None: args.extend( [ '--homedir', self.homedir ] ) + if self.options != None: args.extend( [ '--options', self.options ] ) + if self.comment != None: args.extend( [ '--comment', self.comment ] ) + if self.compress_algo != None: args.extend( [ '--compress-algo', self.compress_algo ] ) + if self.default_key != None: args.extend( [ '--default-key', self.default_key ] ) + + if self.no_options: args.append( '--no-options' ) + if self.armor: args.append( '--armor' ) + if self.textmode: args.append( '--textmode' ) + if self.no_greeting: args.append( '--no-greeting' ) + if self.verbose: args.append( '--verbose' ) + if self.no_verbose: args.append( '--no-verbose' ) + if self.quiet: args.append( '--quiet' ) + if self.batch: args.append( '--batch' ) + if self.always_trust: args.append( '--always-trust' ) + if self.force_v3_sigs: args.append( '--force-v3-sigs' ) + if self.rfc1991: args.append( '--rfc1991' ) + if self.openpgp: args.append( '--openpgp' ) + + for r in self.recipients: args.extend( [ '--recipient', r ] ) + for r in self.encrypt_to: args.extend( [ '--encrypt-to', r ] ) + + return args + + def get_meta_args( self ): + """Get a list of generated meta-arguments""" + args = [] + + if self.meta_pgp_5_compatible: args.extend( [ '--compress-algo', '1', + '--force-v3-sigs' + ] ) + if self.meta_pgp_2_compatible: args.append( '--rfc1991' ) + if not self.meta_interactive: args.extend( [ '--batch', '--no-tty' ] ) + + return args + + +class Process: + """Objects of this class encompass properties of a GnuPG + process spawned by GnuPG.run(). + + # gnupg is a GnuPG object + process = gnupg.run( [ '--decrypt' ], stdout = 1 ) + out = process.handles['stdout'].read() + ... + os.waitpid( process.pid, 0 ) + + Data Attributes + + handles -- This is a map of filehandle-names to + the file handles, if any, that were requested via run() and hence + are connected to the running GnuPG process. Valid names + of this map are only those handles that were requested. + + pid -- The PID of the spawned GnuPG process. + Useful to know, since once should call + os.waitpid() to clean up the process, especially + if multiple calls are made to run(). + """ + + def __init__(self): + self._pipes = {} + self.handles = {} + self.pid = None + self._waited = None + self.closehandles = {} + + def wait(self): + """Wait on the process to exit, allowing for child cleanup. + Will raise an IOError if the process exits non-zero.""" + + e = self.args.wait() + + if self.handles.has_key('stdout'): + if not self.handles['stdout'].closed: + self.handles['stdout'].close() + + if e != 0: + raise IOError, "GnuPG exited non-zero, with code %d" % (e << 8) + +def _run_doctests(): + import doctest, GnuPGInterface + + return doctest.testmod(GnuPGInterface) + +def _run_my_test(): + import GnuPGInterface, subprocess, os + try: +# pobj = subprocess.Popen(["C:\Programmer\GNU\GnuPG\gpg.exe"]) + + plaintext = "Three blind mice" + passphrase = "This is the passphrase" + + gnupg = GnuPGInterface.GnuPG() + gnupg.options.armor = 1 + gnupg.options.meta_interactive = 0 + gnupg.options.extra_args.append('--no-secmem-warning') + + p1 = gnupg.run(['--symmetric'], create_fhs=['stdin', 'stdout', 'passphrase']) + p1.handles['passphrase'].write(passphrase) + p1.handles['passphrase'].close() + + p1.handles['stdin'].write(plaintext) + p1.handles['stdin'].close() + + ciphertext = p1.handles['stdout'].read() + p1.handles['stdout'].close() + + p1.wait() + + gnupg.passphrase = passphrase + p2 = gnupg.run(['--decrypt'], create_fhs=['stdin', 'stdout']) + + p2.handles['stdin'].write(ciphertext) + p2.handles['stdin'].close() + decrypted_plaintext = p2.handles['stdout'].read() + p2.handles['stdout'].close() + p2.wait() + except Exception, e: + pobj = None + +# deprecated +GnuPGInterface = GnuPG + +if __name__ == '__main__': + _run_my_test() + _run_doctests() diff --git a/thirdparty/GnuPGInterface/Homepage.txt b/thirdparty/GnuPGInterface/Homepage.txt new file mode 100644 index 000000000..cd2c88700 --- /dev/null +++ b/thirdparty/GnuPGInterface/Homepage.txt @@ -0,0 +1 @@ +http://py-gnupg.sourceforge.net/ \ No newline at end of file diff --git a/thirdparty/GnuPGInterface/License.txt b/thirdparty/GnuPGInterface/License.txt new file mode 100644 index 000000000..b1e3f5a26 --- /dev/null +++ b/thirdparty/GnuPGInterface/License.txt @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/thirdparty/Readme.txt b/thirdparty/Readme.txt new file mode 100644 index 000000000..e76c61eee --- /dev/null +++ b/thirdparty/Readme.txt @@ -0,0 +1,14 @@ +The files and folders here are not owned by Duplicati. +Each folder has a "License.txt" file that describes the license for that item. +Each folder has a "Homepage.txt" file that contains a link to the page where the files were originally downloaded. + +Duplicity has been modified slightly for use with Windows. +The most work is providing sensible defaults for missing operations. +The way GnuPGInterface.py interacts with gpg.exe was changed. +Due to limitations in Windows, the handles/pipes +cannot be transfered between processes. + +Hopefully these changes will make it back into Duplicity. + +None of the other packages have been modified, and thus does not contain sourcecode. + diff --git a/thirdparty/SQLite/Bin/Homepage.txt b/thirdparty/SQLite/Bin/Homepage.txt new file mode 100644 index 000000000..ea2cb7151 --- /dev/null +++ b/thirdparty/SQLite/Bin/Homepage.txt @@ -0,0 +1 @@ +http://www.sqlite.org/ \ No newline at end of file diff --git a/thirdparty/SQLite/Bin/License.txt b/thirdparty/SQLite/Bin/License.txt new file mode 100644 index 000000000..26eadd0cd --- /dev/null +++ b/thirdparty/SQLite/Bin/License.txt @@ -0,0 +1,34 @@ +SQLite Copyright + +SQLite is in the +Public Domain + +All of the deliverable code in SQLite has been dedicated to the public domain by the authors. All code authors, and representatives of the companies they work for, have signed affidavits dedicating their contributions to the public domain and originals of those signed affidavits are stored in a firesafe at the main offices of Hwaci. Anyone is free to copy, modify, publish, use, compile, sell, or distribute the original SQLite code, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. + +The previous paragraph applies to the deliverable code in SQLite - those parts of the SQLite library that you actually bundle and ship with a larger application. Portions of the documentation and some code used as part of the build process might fall under other licenses. The details here are unclear. We do not worry about the licensing of the documentation and build code so much because none of these things are part of the core deliverable SQLite library. + +All of the deliverable code in SQLite has been written from scratch. No code has been taken from other projects or from the open internet. Every line of code can be traced back to its original author, and all of those authors have public domain dedications on file. So the SQLite code base is clean and is uncontaminated with licensed code from other projects. +Obtaining An Explicit License To Use SQLite + +Even though SQLite is in the public domain and does not require a license, some users want to obtain a license anyway. Some reasons for obtaining a license include: + + * You are using SQLite in a jurisdiction that does not recognize the public domain. + * You are using SQLite in a jurisdiction that does not recognize the right of an author to dedicate their work to the public domain. + * You want to hold a tangible legal document as evidence that you have the legal right to use and distribute SQLite. + * Your legal department tells you that you have to purchase a license. + +If you feel like you really have to purchase a license for SQLite, Hwaci, the company that employs the architect and principal developers of SQLite, will sell you one. +Contributed Code + +In order to keep SQLite completely free and unencumbered by copyright, all new contributors to the SQLite code base are asked to dedicate their contributions to the public domain. If you want to send a patch or enhancement for possible inclusion in the SQLite source tree, please accompany the patch with the following statement: + + The author or authors of this code dedicate any and all copyright interest in this code to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this code under copyright law. + +We are not able to accept patches or changes to SQLite that are not accompanied by a statement such as the above. In addition, if you make changes or enhancements as an employee, then a simple statement such as the above is insufficient. You must also send by surface mail a copyright release signed by a company officer. A signed original of the copyright release should be mailed to: + + Hwaci + 6200 Maple Cove Lane + Charlotte, NC 28269 + USA + +A template copyright release is available in PDF or HTML. You can use this release to make future changes. \ No newline at end of file diff --git a/thirdparty/SQLite/Bin/sqlite3.dll b/thirdparty/SQLite/Bin/sqlite3.dll new file mode 100644 index 000000000..c22d04111 Binary files /dev/null and b/thirdparty/SQLite/Bin/sqlite3.dll differ diff --git a/thirdparty/SQLite/Dll for .Net/Homepage.txt b/thirdparty/SQLite/Dll for .Net/Homepage.txt new file mode 100644 index 000000000..51222311e --- /dev/null +++ b/thirdparty/SQLite/Dll for .Net/Homepage.txt @@ -0,0 +1 @@ +http://sqlite.phxsoftware.com/ \ No newline at end of file diff --git a/thirdparty/SQLite/Dll for .Net/License.txt b/thirdparty/SQLite/Dll for .Net/License.txt new file mode 100644 index 000000000..5096c6573 --- /dev/null +++ b/thirdparty/SQLite/Dll for .Net/License.txt @@ -0,0 +1,6 @@ +ADO.NET 2.0 SQLite Data Provider +Version 1.0.49.0 May 28, 2008 +Using SQLite 3.5.9 +Written by Robert Simpson (robert@blackcastlesoft.com) +Released to the public domain, use at your own risk! +Official provider website: http://sqlite.phxsoftware.com diff --git a/thirdparty/SQLite/Dll for .Net/System.Data.SQLite.DLL b/thirdparty/SQLite/Dll for .Net/System.Data.SQLite.DLL new file mode 100644 index 000000000..6e1905726 Binary files /dev/null and b/thirdparty/SQLite/Dll for .Net/System.Data.SQLite.DLL differ diff --git a/thirdparty/boto/Homepage.txt b/thirdparty/boto/Homepage.txt new file mode 100644 index 000000000..457ac5934 --- /dev/null +++ b/thirdparty/boto/Homepage.txt @@ -0,0 +1 @@ +http://code.google.com/p/boto/ \ No newline at end of file diff --git a/thirdparty/boto/License.txt b/thirdparty/boto/License.txt new file mode 100644 index 000000000..8e2334449 --- /dev/null +++ b/thirdparty/boto/License.txt @@ -0,0 +1,15 @@ +Metadata-Version: 1.0 +Name: boto +Version: 1.3a +Summary: Amazon Web Services Library +Home-page: http://code.google.com/p/boto/ +Author: Mitch Garnaat +Author-email: mitch@garnaat.com +License: MIT +Description: Python interface to Amazon's Web Services. +Platform: Posix; MacOS X; Windows +Classifier: Development Status :: 3 - Alpha +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Topic :: Internet diff --git a/thirdparty/boto/__init__.py b/thirdparty/boto/__init__.py new file mode 100644 index 000000000..6c8bd45b9 --- /dev/null +++ b/thirdparty/boto/__init__.py @@ -0,0 +1,161 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.config import Config, BotoConfigLocations +import os, sys +import logging +import logging.config + +Version = '1.3a' +UserAgent = 'Boto/%s (%s)' % (Version, sys.platform) +config = Config() + +def init_logging(): + for file in BotoConfigLocations: + try: + logging.config.fileConfig(os.path.expanduser(file)) + except: + pass + +log = logging.getLogger('boto') +init_logging() + +# convenience function to set logging to a particular file +def set_file_logger(name, filepath, level=logging.INFO, format_string=None): + global log + if not format_string: + format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" + logger = logging.getLogger(name) + logger.setLevel(level) + fh = logging.FileHandler(filepath) + fh.setLevel(level) + formatter = logging.Formatter(format_string) + fh.setFormatter(formatter) + logger.addHandler(fh) + log = logger + +def connect_sqs(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + @type aws_access_key_id: string + @param aws_access_key_id: Your AWS Access Key ID + + @type aws_secret_access_key: string + @param aws_secret_access_key: Your AWS Secret Access Key + + @rtype: L{SQSConnection} + @return: A connection to Amazon's SQS + """ + from boto.sqs.connection import SQSConnection + return SQSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_s3(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + @type aws_access_key_id: string + @param aws_access_key_id: Your AWS Access Key ID + + @type aws_secret_access_key: string + @param aws_secret_access_key: Your AWS Secret Access Key + + @rtype: L{S3Connection} + @return: A connection to Amazon's S3 + """ + from boto.s3.connection import S3Connection + return S3Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_ec2(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + @type aws_access_key_id: string + @param aws_access_key_id: Your AWS Access Key ID + + @type aws_secret_access_key: string + @param aws_secret_access_key: Your AWS Secret Access Key + + @rtype: L{EC2Connection} + @return: A connection to Amazon's EC2 + """ + from boto.ec2.connection import EC2Connection + return EC2Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_sdb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + @type aws_access_key_id: string + @param aws_access_key_id: Your AWS Access Key ID + + @type aws_secret_access_key: string + @param aws_secret_access_key: Your AWS Secret Access Key + + @rtype: L{SDBConnection} + @return: A connection to Amazon's SDB + """ + from boto.sdb.connection import SDBConnection + return SDBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_fps(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + @type aws_access_key_id: string + @param aws_access_key_id: Your AWS Access Key ID + + @type aws_secret_access_key: string + @param aws_secret_access_key: Your AWS Secret Access Key + + @rtype: L{FPSConnection} + @return: A connection to FPS + """ + from boto.fps.connection import FPSConnection + return FPSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def check_extensions(module_name, module_path): + """ + This function checks for extensions to boto modules. It should be called in the + __init__.py file of all boto modules. See: + http://code.google.com/p/boto/wiki/ExtendModules + + for details. + """ + option_name = '%s_extend' % module_name + version = config.get('Boto', option_name, None) + if version: + dirname = module_path[0] + path = os.path.join(dirname, version) + if os.path.isdir(path): + log.info('extending module %s with: %s' % (module_name, path)) + module_path.insert(0, path) + +_aws_cache = {} + +def _get_aws_conn(service): + global _aws_cache + conn = _aws_cache.get(service) + if not conn: + meth = getattr(sys.modules[__name__], 'connect_'+service) + conn = meth() + _aws_cache[service] = conn + return conn + +def lookup(service, name): + global _aws_cache + conn = _get_aws_conn(service) + obj = _aws_cache.get('.'.join((service,name)), None) + if not obj: + obj = conn.lookup(name) + _aws_cache['.'.join((service,name))] = obj + return obj + diff --git a/thirdparty/boto/connection.py b/thirdparty/boto/connection.py new file mode 100644 index 000000000..117621639 --- /dev/null +++ b/thirdparty/boto/connection.py @@ -0,0 +1,437 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Handles basic connections to AWS +""" + +import base64 +import hmac +import httplib +import socket, errno +import re +import sha +import sys +import time +import urllib, urlparse +import os +import xml.sax +import boto +from boto.exception import AWSConnectionError, BotoClientError, BotoServerError +from boto.resultset import ResultSet +import boto.utils +from boto import config, UserAgent, handler + +PORTS_BY_SECURITY = { True: 443, False: 80 } + +class AWSAuthConnection: + def __init__(self, server, aws_access_key_id=None, + aws_secret_access_key=None, is_secure=True, port=None, + proxy=None, proxy_port=None, proxy_user=None, + proxy_pass=None, debug=0, https_connection_factory=None): + """ + @type server: string + @param server: The server to make the connection to + + @type aws_access_key_id: string + @param aws_access_key_id: AWS Access Key ID (provided by Amazon) + + @type aws_secret_access_key: string + @param aws_secret_access_key: Secret Access Key (provided by Amazon) + + @type is_secure: boolean + @param is_secure: Whether the connection is over SSL + + @type https_connection_factory: list or tuple + @param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. + + @type proxy: + @param proxy: + + @type proxy_port: int + @param proxy_port: The port to use when connecting over a proxy + + @type proxy_user: string + @param proxy_user: The username to connect with on the proxy + + @type proxy_pass: string + @param proxy_pass: The password to use when connection over a proxy. + + @type port: integer + @param port: The port to use to connect + """ + + self.num_retries = 5 + self.is_secure = is_secure + self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) + # define exceptions from httplib that we want to catch and retry + self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) + # define values in socket exceptions we don't want to catch + self.socket_exception_values = (errno.EINTR,) + if https_connection_factory is not None: + self.https_connection_factory = https_connection_factory[0] + self.http_exceptions += https_connection_factory[1] + else: + self.https_connection_factory = None + if (is_secure): + self.protocol = 'https' + else: + self.protocol = 'http' + self.server = server + if debug: + self.debug = debug + else: + self.debug = config.getint('Boto', 'debug', debug) + if port: + self.port = port + else: + self.port = PORTS_BY_SECURITY[is_secure] + self.server_name = '%s:%d' % (server, self.port) + + if aws_access_key_id: + self.aws_access_key_id = aws_access_key_id + elif os.environ.has_key('AWS_ACCESS_KEY_ID'): + self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] + elif config.has_option('Credentials', 'aws_access_key_id'): + self.aws_access_key_id = config.get('Credentials', 'aws_access_key_id') + + if aws_secret_access_key: + self.aws_secret_access_key = aws_secret_access_key + elif os.environ.has_key('AWS_SECRET_ACCESS_KEY'): + self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY'] + elif config.has_option('Credentials', 'aws_secret_access_key'): + self.aws_secret_access_key = config.get('Credentials', 'aws_secret_access_key') + + # initialize an HMAC for signatures, make copies with each request + self.hmac = hmac.new(key=self.aws_secret_access_key, digestmod=sha) + + # cache up to 20 connections + self._cache = boto.utils.LRUCache(20) + self.refresh_http_connection(self.server, self.is_secure) + self._last_rs = None + + def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): + self.proxy = proxy + self.proxy_port = proxy_port + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass + if os.environ.has_key('http_proxy') and not self.proxy: + pattern = re.compile( + '(?:http://)?' \ + '(?:(?P\w+):(?P.*)@)?' \ + '(?P[\w\-\.]+)' \ + '(?::(?P\d+))?' + ) + match = pattern.match(os.environ['http_proxy']) + if match: + self.proxy = match.group('host') + self.proxy_port = match.group('port') + self.proxy_user = match.group('user') + self.proxy_pass = match.group('pass') + else: + if not self.proxy: + self.proxy = config.get_value('Boto', 'proxy', None) + if not self.proxy_port: + self.proxy_port = config.get_value('Boto', 'proxy_port', None) + if not self.proxy_user: + self.proxy_user = config.get_value('Boto', 'proxy_user', None) + if not self.proxy_pass: + self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) + + if not self.proxy_port and self.proxy: + print "http_proxy environment variable does not specify " \ + "a port, using default" + self.proxy_port = self.port + self.use_proxy = (self.proxy != None) + if self.use_proxy and self.is_secure: + raise AWSConnectionError("Unable to provide secure connection through proxy") + + def get_http_connection(self, host, is_secure): + if host is None: + host = '%s:%d' % (self.server, self.port) + cached_name = is_secure and 'https://' or 'http://' + cached_name += host + if cached_name in self._cache: + return self._cache[cached_name] + return self.refresh_http_connection(host, is_secure) + + def refresh_http_connection(self, host, is_secure): + if self.use_proxy: + host = '%s:%d' % (self.proxy, int(self.proxy_port)) + if host is None: + host = '%s:%d' % (self.server, self.port) + boto.log.debug('establishing HTTP connection') + if is_secure: + if self.https_connection_factory: + connection = self.https_connection_factory(host) + else: + connection = httplib.HTTPSConnection(host) + else: + connection = httplib.HTTPConnection(host) + if self.debug > 1: + connection.set_debuglevel(self.debug) + cached_name = is_secure and 'https://' or 'http://' + cached_name += host + if cached_name in self._cache: + boto.log.debug('closing old HTTP connection') + self._cache[cached_name].close() + self._cache[cached_name] = connection + # update self.connection for backwards-compatibility + if host.split(':')[0] == self.server and is_secure == self.is_secure: + self.connection = connection + return connection + + def prefix_proxy_to_path(self, path, host=None): + path = self.protocol + '://' + (host or self.server) + path + return path + + def get_proxy_auth_header(self): + auth = base64.encodestring(self.proxy_user+':'+self.proxy_pass) + return {'Proxy-Authorization': 'Basic %s' % auth} + + def _mexe(self, method, path, data, headers, host=None, sender=None): + """ + mexe - Multi-execute inside a loop, retrying multiple times to handle + transient Internet errors by simply trying again. + Also handles redirects. + + This code was inspired by the S3Utils classes posted to the boto-users + Google group by Larry Bates. Thanks! + """ + boto.log.debug('Method: %s' % method) + boto.log.debug('Path: %s' % path) + boto.log.debug('Data: %s' % data) + boto.log.debug('Headers: %s' % headers) + boto.log.debug('Host: %s' % host) + response = None + body = None + e = None + num_retries = config.getint('Boto', 'num_retries', self.num_retries) + i = 0 + connection = self.get_http_connection(host, self.is_secure) + while i <= num_retries: + try: + if callable(sender): + response = sender(connection, method, path, data, headers) + else: + connection.request(method, path, data, headers) + response = connection.getresponse() + location = response.getheader('location') + if response.status == 500 or response.status == 503: + boto.log.debug('received %d response, retrying in %d seconds' % (response.status, 2**i)) + body = response.read() + elif response.status == 408: + body = response.read() + print '-------------------------' + print ' 4 0 8 ' + print 'path=%s' % path + print body + print '-------------------------' + elif response.status < 300 or response.status >= 400 or \ + not location: + return response + else: + scheme, host, path, params, query, fragment = \ + urlparse.urlparse(location) + if query: + path += '?' + query + boto.log.debug('Redirecting: %s' % scheme + '://' + host + path) + connection = self.get_http_connection(host, + scheme == 'https') + continue + except KeyboardInterrupt: + sys.exit('Keyboard Interrupt') + except self.http_exceptions, e: + boto.log.debug('encountered %s exception, reconnecting' % \ + e.__class__.__name__) + connection = self.refresh_http_connection(host, self.is_secure) + time.sleep(2**i) + i += 1 + # If we made it here, it's because we have exhausted our retries and stil haven't + # succeeded. So, if we have a response object, use it to raise an exception. + # Otherwise, raise the exception that must have already happened. + if response: + raise BotoServerError(response.status, response.reason, body) + elif e: + raise e + else: + raise BotoClientError('Please report this exception as a Boto Issue!') + + def make_request(self, method, path, headers=None, data='', host=None, + auth_path=None, sender=None): + if headers == None: + headers = {'User-Agent' : UserAgent} + else: + headers = headers.copy() + if not headers.has_key('Content-Length'): + headers['Content-Length'] = len(data) + if self.use_proxy: + path = self.prefix_proxy_to_path(path, host) + if self.proxy_user and self.proxy_pass: + headers.update(self.get_proxy_auth_header()) + self.add_aws_auth_header(headers, method, auth_path or path) + return self._mexe(method, path, data, headers, host, sender) + + def add_aws_auth_header(self, headers, method, path): + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + c_string = boto.utils.canonical_string(method, path, headers) + boto.log.debug('Canonical: %s' % c_string) + hmac = self.hmac.copy() + hmac.update(c_string) + b64_hmac = base64.encodestring(hmac.digest()).strip() + headers['Authorization'] = "AWS %s:%s" % (self.aws_access_key_id, b64_hmac) + +class AWSQueryConnection(AWSAuthConnection): + + APIVersion = '' + SignatureVersion = '1' + ResponseError = BotoServerError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=None, debug=0, + https_connection_factory=None): + AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + debug, https_connection_factory) + + def calc_signature_0(self, params): + boto.log.debug('using calc_signature_0') + hmac = self.hmac.copy() + s = params['Action'] + params['Timestamp'] + hmac.update(s) + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + qs = '' + for key in keys: + qs += key + '=' + urllib.quote(unicode(params[key]).encode('utf-8')) + '&' + return (qs, base64.b64encode(hmac.digest())) + + def calc_signature_1(self, params): + boto.log.debug('using calc_signature_1') + hmac = self.hmac.copy() + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + qs = '' + for key in keys: + hmac.update(key) + val = params[key] + if not isinstance(val, str) and not isinstance(val, unicode): + val = str(val) + hmac.update(val) + qs += key + '=' + urllib.quote(unicode(params[key]).encode('utf-8')) + '&' + return (qs, base64.b64encode(hmac.digest())) + + def get_signature(self, params): + if self.SignatureVersion == '1': + t = self.calc_signature_1(params) + elif self.SignatureVersion == '0': + t = self.calc_signature_0(params) + else: + raise BotoClientError('Unknown Signature Version: %s' % self.SignatureVersion) + return t + + def make_request(self, action, params=None, path=None, verb='GET'): + headers = {'User-Agent' : UserAgent} + if path == None: + path = '/' + if params == None: + params = {} + params['Action'] = action + params['Version'] = self.APIVersion + params['AWSAccessKeyId'] = self.aws_access_key_id + params['SignatureVersion'] = self.SignatureVersion + params['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + qs, signature = self.get_signature(params) + qs = path + '?' + qs + 'Signature=' + urllib.quote(signature) + if self.use_proxy: + qs = self.prefix_proxy_to_path(qs) + return self._mexe(verb, qs, None, headers) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params['%s.%d' % (label, i)] = items[i-1] + + # generics + + def get_list(self, action, params, markers, path='/', parent=None): + if not parent: + parent = self + response = self.make_request(action, params, path) + body = response.read() + if response.status == 200: + rs = ResultSet(markers) + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_object(self, action, params, cls, path='/', parent=None): + if not parent: + parent = self + response = self.make_request(action, params, path) + body = response.read() + if response.status == 200: + obj = cls(parent) + h = handler.XmlHandler(obj, parent) + xml.sax.parseString(body, h) + return obj + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_status(self, action, params, path='/', parent=None): + if not parent: + parent = self + response = self.make_request(action, params, path) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs.status + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + diff --git a/thirdparty/boto/contrib/__init__.py b/thirdparty/boto/contrib/__init__.py new file mode 100644 index 000000000..303dbb66c --- /dev/null +++ b/thirdparty/boto/contrib/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/thirdparty/boto/contrib/m2helpers.py b/thirdparty/boto/contrib/m2helpers.py new file mode 100644 index 000000000..82d273051 --- /dev/null +++ b/thirdparty/boto/contrib/m2helpers.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Jon Colverson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Jon Colverson. It provides a couple of helper +functions that allow you to use M2Crypto's implementation of HTTPSConnection +rather than the default version in httplib.py. The main benefit is that +M2Crypto's version verifies the certificate of the server. + +To use this feature, do something like this: + +from boto.ec2.connection import EC2Connection + +ec2 = EC2Connection(ACCESS_KEY_ID, SECRET_ACCESS_KEY, + https_connection_factory=https_connection_factory(cafile=CA_FILE)) + +See http://code.google.com/p/boto/issues/detail?id=57 for more details. +""" +from M2Crypto import SSL +from M2Crypto.httpslib import HTTPSConnection + +def secure_context(cafile=None, capath=None): + ctx = SSL.Context() + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) + if ctx.load_verify_locations(cafile=cafile, capath=capath) != 1: + raise Exception("Couldn't load certificates") + return ctx + +def https_connection_factory(cafile=None, capath=None): + def factory(*args, **kwargs): + return HTTPSConnection( + ssl_context=secure_context(cafile=cafile, capath=capath), + *args, **kwargs) + return (factory, (SSL.SSLError,)) diff --git a/thirdparty/boto/contrib/ymlmessage.py b/thirdparty/boto/contrib/ymlmessage.py new file mode 100644 index 000000000..22e5c627e --- /dev/null +++ b/thirdparty/boto/contrib/ymlmessage.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Chris Moyer +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Chris Moyer. It provides a subclass of the +SQS Message class that supports YAML as the body of the message. + +This module requires the yaml module. +""" +from boto.sqs.message import Message +import yaml + +class YAMLMessage(Message): + """ + The YAMLMessage class provides a YAML compatible message. Encoding and + decoding are handled automaticaly. + + Access this message data like such: + + m.data = [ 1, 2, 3] + m.data[0] # Returns 1 + + This depends on the PyYAML package + """ + + def __init__(self, queue=None, body='', xml_attrs=None): + self.data = None + Message.__init__(self, queue, body) + + def set_body(self, body): + self.data = yaml.load(body) + + def get_body(self): + return yaml.dump(self.data) diff --git a/thirdparty/boto/ec2/__init__.py b/thirdparty/boto/ec2/__init__.py new file mode 100644 index 000000000..449bd162a --- /dev/null +++ b/thirdparty/boto/ec2/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/ec2/address.py b/thirdparty/boto/ec2/address.py new file mode 100644 index 000000000..eec06e712 --- /dev/null +++ b/thirdparty/boto/ec2/address.py @@ -0,0 +1,51 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Address +""" + +class Address: + + def __init__(self, connection=None): + self.connection = connection + self.public_ip = None + self.instance_id = None + + def __repr__(self): + return 'Address:%s' % self.public_ip + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'publicIp': + self.public_ip = value + elif name == 'instanceId': + self.instance_id = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_address(self.public_ip) + + + diff --git a/thirdparty/boto/ec2/connection.py b/thirdparty/boto/ec2/connection.py new file mode 100644 index 000000000..5222ec24b --- /dev/null +++ b/thirdparty/boto/ec2/connection.py @@ -0,0 +1,426 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EC2 service. +""" + +import urllib +import xml.sax +import base64 +import boto +from boto import config +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.ec2.image import Image, ImageAttribute +from boto.ec2.instance import Reservation, Instance, ConsoleOutput +from boto.ec2.keypair import KeyPair +from boto.ec2.address import Address +from boto.ec2.zone import Zone +from boto.ec2.securitygroup import SecurityGroup +from boto.exception import EC2ResponseError + +class EC2Connection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'ec2_version', '2008-02-01') + SignatureVersion = '1' + ResponseError = EC2ResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host='ec2.amazonaws.com', debug=0, + https_connection_factory=None): + """ + Init method to create a new connection to EC2. + + B{Note:} The host argument is overridden by the host specified in the boto configuration file. + """ + if config.has_option('Boto', 'ec2_host'): + host = config.get('Boto', 'ec2_host') + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory) + + # Image methods + + def get_all_images(self, image_ids=None, owners=None, executable_by=None): + """ + Retrieve all the EC2 images available on your account. + + @type image_ids: list + @param image_ids: A list of strings with the image IDs wanted + + @type owners: list + @param owners: A list of owner IDs + + @type executable_by: + @param executable_by: + + @rtype: list + @return: A list of L{Images} + """ + params = {} + if image_ids: + self.build_list_params(params, image_ids, 'ImageId') + if owners: + self.build_list_params(params, owners, 'Owner') + if executable_by: + self.build_list_params(params, executable_by, 'ExecutableBy') + return self.get_list('DescribeImages', params, [('item', Image)]) + + def get_image(self, image_id): + """ + Shortcut method to retrieve a specific image (AMI). + + @type image_id: string + @param image_id: the ID of the Image to retrieve + + @rtype: L{Image} + @return: The EC2 Image specified or None if the image is not found + """ + try: + return self.get_all_images(image_ids=[image_id])[0] + except IndexError: # None of those images available + return None + + def register_image(self, image_location): + params = {'ImageLocation':image_location} + rs = self.get_object('RegisterImage', params, ResultSet) + return rs.imageId + + def deregister_image(self, image_id): + return self.get_status('DeregisterImage', {'ImageId':image_id}) + + # ImageAttribute methods + + def get_image_attribute(self, image_id, attribute='launchPermission'): + """ + Gets an attribute from an image. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-DescribeImageAttribute.html + + @type image_id: string + @param image_id: The Amazon image id for which you want info about + + @type attribute: string + @param attribute: The attribute you need information about + + @rtype: L{ImageAttribute} + @return: An ImageAttribute object representing the value of the attribute requested + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_object('DescribeImageAttribute', params, ImageAttribute) + + def modify_image_attribute(self, image_id, attribute='launchPermission', + operation='add', user_ids=None, groups=None): + """ + Changes an attribute of an image. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-ModifyImageAttribute.html + + @type image_id: string + @param image_id: The image id you wish to change + + @type attribute: string + @param attribute: The attribute you wish to change + + @type operation: string + @param operation: Either add or remove (this is required for changing launchPermissions + + @type user_ids: list + @param user_ids: The Amazon IDs of users to add/remove attributes + + @type groups: list + @param groups: The groups to add/remove attributes + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + return self.get_status('ModifyImageAttribute', params) + + def reset_image_attribute(self, image_id, attribute='launchPermission'): + """ + Rresets an attribute of an AMI to its default value. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-ResetImageAttribute.html + + @type image_id: string + @param image_id: ID of the AMI for which an attribute will be described + + @type attribute: string + @param attribute: The attribute to reset + + @rtype: bool + @return: Whether the operation succeeded or not + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_status('ResetImageAttribute', params) + + # Instance methods + + def get_all_instances(self, instance_ids=None): + """ + Retrieve all the instances associated with your account. + + @type instance_ids: list + @param instance_ids: A list of strings of instance IDs + + @rtype: list + @return: A list of L{Instances} + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('DescribeInstances', params, [('item', Reservation)]) + + def run_instances(self, image_id, min_count=1, max_count=1, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None): + """ + Runs an image on EC2. + + @type image_id: string + @param image_id: The ID of the image to run + + @type instance_type: string + @param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + @rtype: Reservation + @return: The L{Reservation} associated with the request for machines + """ + params = {'ImageId':image_id, + 'MinCount':min_count, + 'MaxCount': max_count} + if key_name: + params['KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'SecurityGroup') + if user_data: + params['UserData'] = base64.b64encode(user_data) + if addressing_type: + params['AddressingType'] = addressing_type + if instance_type: + params['InstanceType'] = instance_type + if placement: + params['Placement.AvailabilityZone'] = placement + return self.get_object('RunInstances', params, Reservation) + + def terminate_instances(self, instance_ids=None): + """ + Terminate the instances specified + + @type instance_ids: list + @param instance_ids: A list of strings of the Instance IDs to terminate + + @rtype: list + @return: A list of the instances terminated + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('TerminateInstances', params, [('item', Instance)]) + + def get_console_output(self, instance_id): + """ + Retrieves the console output for the specified instance. + See http://docs.amazonwebservices.com/AWSEC2/2008-02-01/DeveloperGuide/ApiReference-Query-GetConsoleOutput.html + + @type instance_id: string + @param instance_id: The instance ID of a running instance on the cloud. + + @rtype: L{boto.ec2.instance.ConsoleOutput} + @return: The console output as a ConsoleOutput object + """ + params = {} + self.build_list_params(params, [instance_id], 'InstanceId') + return self.get_object('GetConsoleOutput', params, ConsoleOutput) + + def reboot_instances(self, instance_ids=None): + """ + Reboot the specified instances. + + @type instance_ids: list + @param instance_ids: The instances to terminate and reboot + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_status('RebootInstances', params) + + def confirm_product_instance(self, product_code, instance_id): + params = {'ProductCode' : product_code, + 'InstanceId' : instance_id} + rs = self.get_object('ConfirmProductInstance', params, ResultSet) + return (rs.status, rs.ownerId) + + # Zone methods + + def get_all_zones(self, zones=None): + params = {} + if zones: + self.build_list_params(params, zones, 'ZoneName') + return self.get_list('DescribeAvailabilityZones', params, [('item', Zone)]) + + # Address methods + + def get_all_addresses(self, addresses=None): + params = {} + if addresses: + self.build_list_params(params, addresses, 'PublicIp') + return self.get_list('DescribeAddresses', params, [('item', Address)]) + + def allocate_address(self): + return self.get_object('AllocateAddress', None, Address) + + def release_address(self, public_ip): + params = {'PublicIp' : public_ip} + return self.get_status('ReleaseAddress', params) + + def associate_address(self, instance_id, public_ip): + params = {'InstanceId' : instance_id, 'PublicIp' : public_ip} + return self.get_status('AssociateAddress', params) + + def disassociate_address(self, public_ip): + params = {'PublicIp' : public_ip} + return self.get_status('DisassociateAddress', params) + + # Keypair methods + + def get_all_key_pairs(self, keynames=None): + """ + Get all key pairs associated with your account. + + @type keynames: list + @param keynames: A list of the names of keypairs to retrieve + + @rtype: list + @return: A list of L{KeyPairs} + """ + params = {} + if keynames: + self.build_list_params(params, keynames, 'KeyName') + return self.get_list('DescribeKeyPairs', params, [('item', KeyPair)]) + + def get_key_pair(self, keyname): + """ + Convenience method to retrieve a specific keypair (KeyPair). + + @type image_id: string + @param image_id: the ID of the Image to retrieve + + @rtype: L{KeyPair} + @return: The KeyPair specified or None if it is not found + """ + try: + return self.get_all_key_pairs(keynames=[keyname])[0] + except IndexError: # None of those images available + return None + + def create_key_pair(self, key_name): + """ + Create a new key pair for your account. + + @type key_name: string + @param key_name: The name of the new keypair + + @rtype: KeyPair + @return: The newly created L{KeyPair} + """ + params = {'KeyName':key_name} + return self.get_object('CreateKeyPair', params, KeyPair) + + def delete_key_pair(self, key_name): + """ + Delete a key pair from your account. + + @type key_name: string + @param key_name: The name of the keypair to delete + """ + params = {'KeyName':key_name} + return self.get_status('DeleteKeyPair', params) + + # SecurityGroup methods + + def get_all_security_groups(self, groupnames=None): + params = {} + if groupnames: + self.build_list_params(params, groupnames, 'GroupName') + return self.get_list('DescribeSecurityGroups', params, [('item', SecurityGroup)]) + + def create_security_group(self, name, description): + params = {'GroupName':name, 'GroupDescription':description} + group = self.get_object('CreateSecurityGroup', params, SecurityGroup) + group.name = name + group.description = description + return group + + def delete_security_group(self, name): + params = {'GroupName':name} + return self.get_status('DeleteSecurityGroup', params) + + def authorize_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + if ip_protocol: + params['IpProtocol'] = ip_protocol + if from_port: + params['FromPort'] = from_port + if to_port: + params['ToPort'] = to_port + if cidr_ip: + params['CidrIp'] = urllib.quote(cidr_ip) + return self.get_status('AuthorizeSecurityGroupIngress', params) + + def revoke_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + if ip_protocol: + params['IpProtocol'] = ip_protocol + if from_port: + params['FromPort'] = from_port + if to_port: + params['ToPort'] = to_port + if cidr_ip: + params['CidrIp'] = cidr_ip + return self.get_status('RevokeSecurityGroupIngress', params) + diff --git a/thirdparty/boto/ec2/image.py b/thirdparty/boto/ec2/image.py new file mode 100644 index 000000000..89bed5f63 --- /dev/null +++ b/thirdparty/boto/ec2/image.py @@ -0,0 +1,174 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Image +""" + +class Image: + + def __init__(self, connection=None): + self.connection = connection + self.id = None + self.location = None + self.state = None + self.ownerId = None + self.is_public = False + self.product_codes = [] + + def __repr__(self): + return 'Image:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'imageId': + self.id = value + elif name == 'imageLocation': + self.location = value + elif name == 'imageState': + self.state = value + elif name == 'imageOwnerId': + self.ownerId = value + elif name == 'isPublic': + if value == 'false': + self.is_public = False + elif value == 'true': + self.is_public = True + else: + raise Exception( + 'Unexpected value of isPublic %s for image %s'%( + value, + self.id + ) + ) + elif name == 'productCode': + self.product_codes.append(value) + else: + setattr(self, name, value) + + def run(self, min_count=1, max_count=1, key_name=None, + security_groups=None, user_data=None, + addressing_type=None, instance_type='m1.small', placement=None): + """ + Runs this instance. + + @type min_count: int + @param min_count: The minimum number of instances to start + + @type max_count: int + @param max_count: The maximum number of instances to start + + @type key_name: string + @param key_name: The keypair to run this instance with. + + @type security_groups: + @param security_groups: + + @type user_data: + @param user_data: + + @type addressing_type: + @param daddressing_type: + + @type instance_type: string + @param instance_type: The type of instance to run (m1.small, m1.large, m1.xlarge) + + @type placement: + @param placement: + """ + return self.connection.run_instances(self.id, min_count, max_count, + key_name, security_groups, + user_data, addressing_type, + instance_type, placement) + + def deregister(self): + return self.connection.deregister_image(self.id) + + def get_launch_permissions(self): + img_attrs = self.connection.get_image_attribute(self.id, + 'launchPermission') + return img_attrs.attrs + + def set_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'add', + user_ids, + group_names) + + def remove_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'remove', + user_ids, + group_names) + + def reset_launch_attributes(self): + return self.connection.reset_image_attribute(self.id, + 'launchPermission') + + def get_kernel(self): + img_attrs =self.connection.get_image_attribute(self.id, 'kernel') + return img_attrs.kernel + + def get_ramdisk(self): + img_attrs = self.connection.get_image_attribute(self.id, 'ramdisk') + return img_attrs.ramdisk + +class ImageAttribute: + + def __init__(self, parent=None): + self.name = None + self.kernel = None + self.ramdisk = None + self.attrs = {} + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'launchPermission': + self.name = 'launch_permission' + elif name == 'productCodes': + self.name = 'product_codes' + self.attrs[self.name] = [] + elif name == 'group': + if self.attrs.has_key('groups'): + self.attrs['groups'].append(value) + else: + self.attrs['groups'] = [value] + elif name == 'userId': + if self.attrs.has_key('user_ids'): + self.attrs['user_ids'].append(value) + else: + self.attrs['user_ids'] = [value] + elif name == 'imageId': + self.image_id = value + elif name == 'productCode': + self.attrs['product_codes'].append(value) + elif name == 'kernel': + self.kernel = value + elif name == 'ramdisk': + self.ramdisk = value + else: + setattr(self, name, value) diff --git a/thirdparty/boto/ec2/instance.py b/thirdparty/boto/ec2/instance.py new file mode 100644 index 000000000..727bc9374 --- /dev/null +++ b/thirdparty/boto/ec2/instance.py @@ -0,0 +1,200 @@ +# Copyright (c) 2006, 2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Instance +""" + +from boto.resultset import ResultSet +from boto.ec2.address import Address +import base64 + +class Reservation: + + def __init__(self, connection=None): + self.connection = connection + self.id = None + self.owner_id = None + self.groups = [] + self.instances = [] + + def __repr__(self): + return 'Reservation:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'instancesSet': + self.instances = ResultSet([('item', Instance)]) + return self.instances + elif name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + else: + return None + + def endElement(self, name, value, connection): + if name == 'reservationId': + self.id = value + elif name == 'ownerId': + self.owner_id = value + else: + setattr(self, name, value) + + def stop_all(self): + for instance in self.instances: + instance.stop() + +class Instance: + + def __init__(self, connection=None): + self.connection = connection + self.id = None + self.dns_name = None + self.public_dns_name = None + self.private_dns_name = None + self.state = None + self.state_code = None + self.key_name = None + self.shutdown_state = None + self.previous_state = None + self.instance_type = None + self.launch_time = None + self.image_id = None + self.placement = None + self.kernel = None + self.ramdisk = None + + def __repr__(self): + return 'Instance:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.id = value + elif name == 'imageId': + self.image_id = value + elif name == 'dnsName' or name == 'publicDnsName': + self.dns_name = value # backwards compatibility + self.public_dns_name = value + elif name == 'privateDnsName': + self.private_dns_name = value + elif name == 'keyName': + self.key_name = value + elif name == 'amiLaunchIndex': + self.ami_launch_index = value + elif name == 'shutdownState': + self.shutdown_state = value + elif name == 'previousState': + self.previous_state = value + elif name == 'name': + self.state = value + elif name == 'code': + self.state_code = int(value) + elif name == 'instanceType': + self.instance_type = value + elif name == 'launchTime': + self.launch_time = value + elif name == 'availabilityZone': + self.placement = value + elif name == 'kernelId': + self.kernel = value + elif name == 'ramdiskId': + self.ramdisk = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.updated = updated + if hasattr(updated, 'dns_name'): + self.dns_name = updated.dns_name + self.public_dns_name = updated.dns_name + if hasattr(updated, 'private_dns_name'): + self.private_dns_name = updated.private_dns_name + if hasattr(updated, 'ami_launch_index'): + self.ami_launch_index = updated.ami_launch_index + self.shutdown_state = updated.shutdown_state + self.previous_state = updated.previous_state + if hasattr(updated, 'state'): + self.state = updated.state + else: + self.state = None + if hasattr(updated, 'state_code'): + self.state_code = updated.state_code + else: + self.state_code = None + + def update(self): + rs = self.connection.get_all_instances([self.id]) + if len(rs) > 0: + self._update(rs[0].instances[0]) + return self.state + + def stop(self): + rs = self.connection.terminate_instances([self.id]) + self._update(rs[0]) + + def reboot(self): + return self.connection.reboot_instances([self.id]) + + def get_console_output(self): + return self.connection.get_console_output(self.id) + + def confirm_product(self, product_code): + return self.connection.confirm_product_instance(self.id, product_code) + + def use_ip(self, ip_address): + if isinstance(ip_address, Address): + ip_address = ip_address.public_ip + return self.connection.associate_address(self.id, ip_address) + +class Group: + + def __init__(self, parent=None): + self.id = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'groupId': + self.id = value + else: + setattr(self, name, value) + +class ConsoleOutput: + + def __init__(self, parent=None): + self.parent = parent + self.instance_id = None + self.timestamp = None + self.comment = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.instance_id = value + elif name == 'output': + self.output = base64.b64decode(value) + else: + setattr(self, name, value) diff --git a/thirdparty/boto/ec2/keypair.py b/thirdparty/boto/ec2/keypair.py new file mode 100644 index 000000000..172a80c95 --- /dev/null +++ b/thirdparty/boto/ec2/keypair.py @@ -0,0 +1,54 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Keypair +""" + +class KeyPair: + + def __init__(self, connection=None): + self.connection = connection + self.name = None + self.fingerprint = None + self.material = None + + def __repr__(self): + return 'KeyPair:%s' % self.name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'keyName': + self.name = value + elif name == 'keyFingerprint': + self.fingerprint = value + elif name == 'keyMaterial': + self.material = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_key_pair(self.name) + + + diff --git a/thirdparty/boto/ec2/securitygroup.py b/thirdparty/boto/ec2/securitygroup.py new file mode 100644 index 000000000..ba951c075 --- /dev/null +++ b/thirdparty/boto/ec2/securitygroup.py @@ -0,0 +1,203 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Security Group +""" + +class SecurityGroup: + + def __init__(self, connection=None, owner_id=None, + name=None, description=None): + self.connection = connection + self.owner_id = owner_id + self.name = name + self.description = description + self.rules = [] + + def __repr__(self): + return 'SecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'item': + self.rules.append(IPPermissions(self)) + return self.rules[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'ownerId': + self.owner_id = value + elif name == 'groupName': + self.name = value + elif name == 'groupDescription': + self.description = value + elif name == 'ipRanges': + pass + elif name == 'return': + if value == 'false': + self.status = False + elif value == 'true': + self.status = True + else: + raise Exception( + 'Unexpected value of status %s for image %s'%( + value, + self.id + ) + ) + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_security_group(self.name) + + def add_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + rule = IPPermissions(self) + rule.ip_protocol = ip_protocol + rule.from_port = from_port + rule.to_port = to_port + self.rules.append(rule) + rule.add_grant(src_group_name, src_group_owner_id, cidr_ip) + + def remove_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + target_rule = None + for rule in self.rules: + if rule.ip_protocol == ip_protocol: + if rule.from_port == from_port: + if rule.to_port == to_port: + target_rule = rule + target_grant = None + for grant in rule.grants: + if grant.group_name == src_group_name: + if grant.user_id == src_group_owner_id: + if grant.cidr_ip == cidr_ip: + target_grant = grant + if target_grant: + rule.grants.remove(target_grant) + if len(rule.grants) == 0: + self.rules.remove(target_rule) + + def authorize(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + if src_group: + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.authorize_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.add_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + + def revoke(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + if src_group: + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.revoke_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.remove_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + +class IPPermissions: + + def __init__(self, parent=None): + self.parent = parent + self.ip_protocol = None + self.from_port = None + self.to_port = None + self.grants = [] + + def __repr__(self): + return 'IPPermissions:%s(%s-%s)' % (self.ip_protocol, + self.from_port, self.to_port) + + def startElement(self, name, attrs, connection): + if name == 'item': + self.grants.append(GroupOrCIDR(self)) + return self.grants[-1] + return None + + def endElement(self, name, value, connection): + if name == 'ipProtocol': + self.ip_protocol = value + elif name == 'fromPort': + self.from_port = value + elif name == 'toPort': + self.to_port = value + else: + setattr(self, name, value) + + def add_grant(self, user_id=None, group_name=None, cidr_ip=None): + grant = GroupOrCIDR(self) + grant.user_id = user_id + grant.group_name = group_name + grant.cidr_ip = cidr_ip + self.grants.append(grant) + return grant + +class GroupOrCIDR: + + def __init__(self, parent=None): + self.user_id = None + self.group_name = None + self.cidr_ip = None + + def __repr__(self): + if self.cidr_ip: + return '%s' % self.cidr_ip + else: + return '%s-%s' % (self.group_name, self.user_id) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'userId': + self.user_id = value + elif name == 'groupName': + self.group_name = value + if name == 'cidrIp': + self.cidr_ip = value + else: + setattr(self, name, value) + diff --git a/thirdparty/boto/ec2/zone.py b/thirdparty/boto/ec2/zone.py new file mode 100644 index 000000000..f8bf3ad78 --- /dev/null +++ b/thirdparty/boto/ec2/zone.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Availability Zone +""" + +class Zone: + + def __init__(self, connection=None): + self.connection = connection + self.name = None + self.state = None + + def __repr__(self): + return 'Zone:%s' % self.name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'zoneName': + self.name = value + elif name == 'zoneState': + self.state = value + else: + setattr(self, name, value) + + + + diff --git a/thirdparty/boto/exception.py b/thirdparty/boto/exception.py new file mode 100644 index 000000000..e942ea989 --- /dev/null +++ b/thirdparty/boto/exception.py @@ -0,0 +1,114 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Exception classes - Subclassing allows you to check for specific errors +""" + +class BotoClientError(Exception): + """ + General Boto Client error (error accessing AWS) + """ + + def __init__(self, reason): + self.reason = reason + + def __repr__(self): + return 'S3Error: %s' % self.reason + + def __str__(self): + return 'S3Error: %s' % self.reason + +class SDBPersistenceError(Exception): + + pass + +class S3PermissionsError(BotoClientError): + """ + Permissions error when accessing a bucket or key on S3. + """ + pass + +class BotoServerError(Exception): + + def __init__(self, status, reason, body=''): + self.status = status + self.reason = reason + self.body = body + + def __repr__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + + def __str__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + +class S3CreateError(BotoServerError): + """ + Error creating a bucket or key on S3. + """ + pass + +class S3CopyError(BotoServerError): + """ + Error copying a key on S3. + """ + pass + +class SQSError(BotoServerError): + """ + General Error on Simple Queue Service. + """ + pass + +class S3ResponseError(BotoServerError): + """ + Error in response from S3. + """ + pass + +class EC2ResponseError(BotoServerError): + """ + Error in response from EC2. + """ + pass + +class SDBResponseError(BotoServerError): + """ + Error in respones from SDB. + """ + pass + +class AWSConnectionError(BotoClientError): + """ + General error connecting to Amazon Web Services. + """ + pass + +class S3DataError(BotoClientError): + """ + Error receiving data from S3. + """ + pass + +class FPSResponseError(BotoServerError): + pass diff --git a/thirdparty/boto/fps/__init__.py b/thirdparty/boto/fps/__init__.py new file mode 100644 index 000000000..2f44483df --- /dev/null +++ b/thirdparty/boto/fps/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2008, Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/fps/connection.py b/thirdparty/boto/fps/connection.py new file mode 100644 index 000000000..e460931ab --- /dev/null +++ b/thirdparty/boto/fps/connection.py @@ -0,0 +1,171 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import xml.sax +import uuid +import boto +import boto.utils +import urllib +from boto import handler +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.exception import FPSResponseError + +class FPSConnection(AWSQueryConnection): + + APIVersion = '2007-01-08' + SignatureVersion = '1' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + host='fps.sandbox.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + host, debug, https_connection_factory) + + def install_payment_instruction(self, instruction, token_type="Unrestricted", transaction_id=None): + """ + InstallPaymentInstruction + instruction: The PaymentInstruction to send, for example: + MyRole=='Caller' orSay 'Roles do not match'; + token_type: Defaults to "Unrestricted" + transaction_id: Defaults to a new ID + """ + + if(transaction_id == None): + transaction_id = uuid.uuid4() + params = {} + params['PaymentInstruction'] = instruction + params['TokenType'] = token_type + params['CallerReference'] = transaction_id + response = self.make_request("InstallPaymentInstruction", params) + return response + + def install_caller_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a caller + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the caller token. + """ + response = self.install_payment_instruction("MyRole=='Caller';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + caller_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "caller_token", caller_token) + except(IOError): + boto.config.save_user_option("FPS", "caller_token", caller_token) + return caller_token + else: + raise FPSResponseError(response.status, respons.reason, body) + + def install_recipient_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a Recipient + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the recipient token. + """ + response = self.install_payment_instruction("MyRole=='Recipient';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + recipient_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "recipient_token", recipient_token) + except(IOError): + boto.config.save_user_option("FPS", "recipient_token", recipient_token) + + return recipient_token + else: + raise FPSResponseError(response.status, respons.reason, body) + + def make_url(self, returnURL, paymentReason, pipelineName, **params): + """ + Generate the URL with the signature required for a transaction + """ + params['callerKey'] = str(self.aws_access_key_id) + params['returnURL'] = str(returnURL) + params['paymentReason'] = str(paymentReason) + params['pipelineName'] = pipelineName + + if(not params.has_key('callerReference')): + params['callerReference'] = str(uuid.uuid4()) + + url = "" + keys = params.keys() + keys.sort() + for k in keys: + url += "&%s=%s" % (k, urllib.quote_plus(str(params[k]))) + + url = "/cobranded-ui/actions/start?%s" % ( url[1:]) + signature= boto.utils.encode(self.aws_secret_access_key, url, True) + return "https://authorize.payments-sandbox.amazon.com%s&awsSignature=%s" % (url, signature) + + def make_payment(self, amount, sender_token, charge_fee_to="Recipient", reference=None, senderReference=None, recipientReference=None, senderDescription=None, recipientDescription=None, callerDescription=None, metadata=None, transactionDate=None): + """ + Make a payment transaction + You must specify the amount and the sender token. + """ + params = {} + params['RecipientTokenId'] = boto.config.get("FPS", "recipient_token") + params['CallerTokenId'] = boto.config.get("FPS", "caller_token") + params['SenderTokenId'] = sender_token + params['TransactionAmount.Amount'] = str(amount) + params['TransactionAmount.CurrencyCode'] = "USD" + params['ChargeFeeTo'] = charge_fee_to + + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(senderReference != None): + params['SenderReference'] = senderReference + if(recipientReference != None): + params['RecipientReference'] = recipientReference + if(senderDescription != None): + params['SenderDescription'] = senderDescription + if(recipientDescription != None): + params['RecipientDescription'] = recipientDescription + if(callerDescription != None): + params['CallerDescription'] = callerDescription + if(metadata != None): + params['MetaData'] = metadata + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(reference == None): + reference = uuid.uuid4() + params['CallerReference'] = reference + + response = self.make_request("Pay", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) diff --git a/thirdparty/boto/handler.py b/thirdparty/boto/handler.py new file mode 100644 index 000000000..525f9c9a6 --- /dev/null +++ b/thirdparty/boto/handler.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax + +class XmlHandler(xml.sax.ContentHandler): + + def __init__(self, root_node, connection): + self.connection = connection + self.nodes = [('root', root_node)] + self.current_text = '' + + def startElement(self, name, attrs): + self.current_text = '' + new_node = self.nodes[-1][1].startElement(name, attrs, self.connection) + if new_node != None: + self.nodes.append((name, new_node)) + + def endElement(self, name): + self.nodes[-1][1].endElement(name, self.current_text, self.connection) + if self.nodes[-1][0] == name: + self.nodes.pop() + self.current_text = '' + + def characters(self, content): + self.current_text += content + + diff --git a/thirdparty/boto/mapreduce/__init__.py b/thirdparty/boto/mapreduce/__init__.py new file mode 100644 index 000000000..ac3ddc447 --- /dev/null +++ b/thirdparty/boto/mapreduce/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/mapreduce/lqs.py b/thirdparty/boto/mapreduce/lqs.py new file mode 100644 index 000000000..fc76e50f8 --- /dev/null +++ b/thirdparty/boto/mapreduce/lqs.py @@ -0,0 +1,152 @@ +import SocketServer, os, datetime, sys, random, time +import simplejson + +class LQSCommand: + + def __init__(self, line): + self.raw_line = line + self.line = self.raw_line.strip() + l = self.line.split(' ') + self.name = l[0] + if len(l) > 1: + self.args = [arg for arg in l[1:] if arg] + else: + self.args = [] + +class LQSMessage(dict): + + def __init__(self, item=None, args=None, jsonvalue=None): + dict.__init__(self) + if jsonvalue: + self.decode(jsonvalue) + else: + self['id'] = '%d_%d' % (int(time.time()), int(random.random()*1000000)) + self['item'] = item + self['args'] = args + + def encode(self): + return simplejson.dumps(self) + + def decode(self, value): + self.update(simplejson.loads(value)) + + def is_empty(self): + if self['item'] == None: + return True + return False + +class LQSServer(SocketServer.UDPServer): + + PORT = 5151 + TIMEOUT = 30 + MAXSIZE = 8192 + + def __init__(self, server_address, RequestHandlerClass, iterator, args=None): + server_address = (server_address, self.PORT) + SocketServer.UDPServer.__init__(self, server_address, RequestHandlerClass) + self.count = 0 + self.iterator = iterator + self.args = args + self.start = datetime.datetime.now() + self.end = None + self.extant = [] + +class LQSHandler(SocketServer.DatagramRequestHandler): + + def get_cmd(self): + return LQSCommand(self.rfile.readline()) + + def build_msg(self): + if not self.server.iterator: + return LQSMessage(None) + try: + item = self.server.iterator.next() + msg = LQSMessage(item, self.server.args) + return msg + except StopIteration: + self.server.iterator = None + return LQSMessage(None) + + def respond(self, msg): + self.wfile.write(msg.encode()) + + def check_extant(self): + if len(self.server.extant) == 0 and not self.server.iterator: + self.server.end = datetime.datetime.now() + delta = self.server.end - self.server.start + print 'Total Processing Time: %s' % delta + print 'Total Messages Processed: %d' % self.server.count + + def do_debug(self, cmd): + args = {'extant' : self.server.extant, + 'count' : self.server.count} + msg = LQSMessage('debug', args) + self.respond(msg) + + def do_next(self, cmd): + out_msg = self.build_msg() + if not out_msg.is_empty(): + self.server.count += 1 + self.server.extant.append(out_msg['id']) + self.respond(out_msg) + + def do_delete(self, cmd): + if len(cmd.args) != 1: + self.error(cmd, 'delete command requires message id') + else: + mid = cmd.args[0] + try: + self.server.extant.remove(mid) + except ValueError: + self.error(cmd, 'message id not found') + args = {'deleted' : True} + msg = LQSMessage(mid, args) + self.respond(msg) + self.check_extant() + + def error(self, cmd, error_msg=None): + args = {'error_msg' : error_msg, + 'cmd_name' : cmd.name, + 'cmd_args' : cmd.args} + msg = LQSMessage('error', args) + self.respond(msg) + + def do_stop(self, cmd): + sys.exit(0) + + def handle(self): + cmd = self.get_cmd() + if hasattr(self, 'do_%s' % cmd.name): + method = getattr(self, 'do_%s' % cmd.name) + method(cmd) + else: + self.error(cmd, 'unrecognized command') + +class PersistHandler(LQSHandler): + + def build_msg(self): + if not self.server.iterator: + return LQSMessage(None) + try: + obj = self.server.iterator.next() + msg = LQSMessage(obj.id, self.server.args) + return msg + except StopIteration: + self.server.iterator = None + return LQSMessage(None) + +def test_file(path, args=None): + l = os.listdir(path) + if not args: + args = {} + args['path'] = path + s = LQSServer('', LQSHandler, iter(l), args) + print "Awaiting UDP messages on port %d" % s.PORT + s.serve_forever() + +def test_simple(n): + l = range(0, n) + s = LQSServer('', LQSHandler, iter(l), None) + print "Awaiting UDP messages on port %d" % s.PORT + s.serve_forever() + diff --git a/thirdparty/boto/mapreduce/partitiondb.py b/thirdparty/boto/mapreduce/partitiondb.py new file mode 100644 index 000000000..6443d0293 --- /dev/null +++ b/thirdparty/boto/mapreduce/partitiondb.py @@ -0,0 +1,172 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import random, time, os, datetime +import boto +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import * + +class Identifier(object): + + _hex_digits = '0123456789abcdef' + + @classmethod + def gen(cls, prefix): + suffix = '' + for i in range(0,8): + suffix += random.choice(cls._hex_digits) + return ts + '-' + suffix + +class Version(SDBObject): + + name = StringProperty() + pdb = ObjectProperty(ref_class=SDBObject) + date = DateTimeProperty() + + def __init__(self, id=None, manager=None): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = Identifier.gen('v') + self.date = datetime.datetime.now() + print 'created Version %s' % self.name + + def partitions(self): + """ + Return an iterator containing all Partition objects related to this Version. + + @rtype: iterator of L{Partitions} + @return: The Partitions in this Version + """ + return self.get_related_objects('version', Partition) + + def add_partition(self, name=None): + """ + Add a new Partition to this Version. + + @type name: string + @param name: The name of the new Partition (optional) + + @rtype: L{Partition} + @return: The new Partition object + """ + p = Partition(manager=self.manager, name=name) + p.version = self + p.pdb = self.pdb + p.save() + return p + + def get_s3_prefix(self): + if not self.pdb: + raise ValueError, 'pdb attribute must be set to compute S3 prefix' + return self.pdb.get_s3_prefix() + self.name + '/' + +class PartitionDB(SDBObject): + + name = StringProperty() + bucket_name = StringProperty() + versions = ObjectListProperty(ref_class=Version) + + def __init__(self, id=None, manager=None, name='', bucket_name=''): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = name + self.bucket_name = bucket_name + + def get_s3_prefix(self): + return self.name + '/' + + def add_version(self): + """ + Add a new Version to this PartitionDB. The newly added version becomes the + current version. + + @rtype: L{Version} + @return: The newly created Version object. + """ + v = Version() + v.pdb = self + v.save() + self.versions.append(v) + return v + + def revert(self): + """ + Revert to the previous version of this PartitionDB. The current version is removed from the + list of Versions and the Version immediately preceeding it becomes the current version. + Note that this method does not delete the Version object or any Partitions related to the + Version object. + + @rtype: L{Version} + @return: The previous current Version object. + """ + v = self.current_version() + if v: + self.versions.remove(v) + return v + + def current_version(self): + """ + Get the currently active Version of this PartitionDB object. + + @rtype: L{Version} + @return: The current Version object or None if there are no Versions associated + with this PartitionDB object. + """ + if self.versions: + if len(self.versions) > 0: + return self.versions[-1] + return None + +class Partition(SDBObject): + + def __init__(self, id=None, manager=None, name=None): + SDBObject.__init__(self, id, manager) + if id == None: + self.name = name + + name = StringProperty() + version = ObjectProperty(ref_class=Version) + pdb = ObjectProperty(ref_class=PartitionDB) + data = S3KeyProperty() + + def get_key_name(self): + return self.version.get_s3_prefix() + self.name + + def upload(self, path, bucket_name=None): + if not bucket_name: + bucket_name = self.version.pdb.bucket_name + s3 = self.manager.get_s3_connection() + bucket = s3.lookup(bucket_name) + directory, filename = os.path.split(path) + self.name = filename + key = bucket.new_key(self.get_key_name()) + key.set_contents_from_filename(path) + self.data = key + self.save() + + def delete(self): + if self.data: + self.data.delete() + SDBObject.delete(self) + + + diff --git a/thirdparty/boto/mapreduce/pdb_delete b/thirdparty/boto/mapreduce/pdb_delete new file mode 100644 index 000000000..b7af9cc06 --- /dev/null +++ b/thirdparty/boto/mapreduce/pdb_delete @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, PersistHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, revive_object_from_id + +USAGE = """ + SYNOPSIS + %prog [options] [command] + DESCRIPTION + Delete a PartitionDB and all related data in SimpleDB and S3. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + while m['item']: + print 'Deleting: %s' % m['item'] + obj = revive_object_from_id(m['item'], manager=self.manager) + obj.delete() + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, pdb_name, domain_name=None): + self.pdb_name = pdb_name + self.manager = get_manager(domain_name) + self.pdb = PartitionDB.get(name=self.pdb_name) + self.serve() + + def serve(self): + args = {'pdb_id' : self.pdb.id} + rs = self.pdb.get_related_objects('pdb') + self.pdb.delete() + s = LQSServer('', PersistHandler, rs, args) + s.serve_forever() + +class Delete: + + Commands = {'client' : 'Start a Delete client', + 'server' : 'Start a Delete server'} + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=5) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.pdb_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s ' % (self.prog_name, self.options.pdb_name) + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + delete = Delete() + delete.main() diff --git a/thirdparty/boto/mapreduce/pdb_describe b/thirdparty/boto/mapreduce/pdb_describe new file mode 100644 index 000000000..d0fa86c63 --- /dev/null +++ b/thirdparty/boto/mapreduce/pdb_describe @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import sys +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, get_domain + +USAGE = """ + SYNOPSIS + %prog [options] + DESCRIPTION + List and describe your PartitionDBs. + Called with no options, all PartitionDB objects defined in your default + domain (as specified in the "default_domain" option in the "[Persist]" + section of your boto config file) will be listed. + When called with a particular PartitionDB name (using -p option) all + Version objects of that PartitionDB object will be listed. + When called with the -p option and a particular Version name specified + (using the -v option) all Partitions in that Version object will be listed. +""" +class Describe: + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-entries', action='store', type='int', + help='maximum number of entries to print (default 100)') + self.parser.set_defaults(num_entries=100) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB to describe') + self.parser.add_option('-v', '--version-name', action='store', type='string', + help='name of the PDB Version to describe') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def describe_all(self): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDBs:' + rs = PartitionDB.list() + i = 0 + for pdb in rs: + print '%s\t%s\t%s' % (pdb.id, pdb.name, pdb.bucket_name) + i += 1 + if i == self.options.num_entries: + break + + def describe_pdb(self, pdb_name): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDB: %s' % pdb_name + print 'Versions:' + try: + pdb = PartitionDB.get(name=pdb_name) + i = 0 + for v in pdb.versions: + if v.date: + ds = v.date.isoformat() + else: + ds = 'unknown' + print '%s\t%s\t%s' % (v.id, v.name, ds) + i += 1 + if i == self.options.num_entries: + break + cv = pdb.current_version() + if cv: + print 'Current Version: %s' % cv.name + else: + print 'Current Version: None' + except SDBPersistenceError: + self.parser.error('pdb_name (%s) unknown' % pdb_name) + + def describe_version(self, pdb_name, version_name): + print 'Using SimpleDB Domain: %s' % get_domain() + print 'PDB: %s' % pdb_name + print 'Version: %s' % version_name + print 'Partitions:' + try: + pdb = PartitionDB.get(name=pdb_name) + for v in pdb.versions: + if v.name == version_name: + i = 0 + for p in v.partitions(): + print '%s\t%s' % (p.id, p.name) + i += 1 + if i == self.options.num_entries: + break + except SDBPersistenceError: + self.parser.error('pdb_name (%s) unknown' % pdb_name) + + def main(self): + self.options, self.args = self.parser.parse_args() + self.manager = get_manager(self.options.domain_name) + + if self.options.pdb_name: + if self.options.version_name: + self.describe_version(self.options.pdb_name, self.options.version_name) + else: + self.describe_pdb(self.options.pdb_name) + else: + self.describe_all() + +if __name__ == "__main__": + describe = Describe() + describe.main() diff --git a/thirdparty/boto/mapreduce/pdb_revert b/thirdparty/boto/mapreduce/pdb_revert new file mode 100644 index 000000000..daffeef1e --- /dev/null +++ b/thirdparty/boto/mapreduce/pdb_revert @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, PersistHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager + +USAGE = """ + SYNOPSIS + %prog [options] [command] + DESCRIPTION + Revert to the previous Version in a PartitionDB. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + while m['item']: + print 'Deleting: %s' % m['item'] + p = Partition(id=m['item'], manager=self.manager) + p.delete() + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, pdb_name, domain_name=None): + self.pdb_name = pdb_name + self.manager = get_manager(domain_name) + self.pdb = PartitionDB.get(name=self.pdb_name) + self.serve() + + def serve(self): + v = self.pdb.revert() + args = {'v_id' : v.id} + rs = v.partitions() + s = LQSServer('', PersistHandler, rs, args) + s.serve_forever() + +class Revert: + + Commands = {'client' : 'Start a Revert client', + 'server' : 'Start a Revert server'} + + def __init__(self): + self.parser = OptionParser(usage=USAGE) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=5) + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.pdb_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s ' % (self.prog_name, self.options.pdb_name) + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + revert = Revert() + revert.main() diff --git a/thirdparty/boto/mapreduce/pdb_upload b/thirdparty/boto/mapreduce/pdb_upload new file mode 100644 index 000000000..1ca2b6d39 --- /dev/null +++ b/thirdparty/boto/mapreduce/pdb_upload @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import queuetools, os, signal, sys +import subprocess +import time +from optparse import OptionParser +from boto.mapreduce.partitiondb import PartitionDB, Partition, Version +from lqs import LQSServer, LQSHandler +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager + +USAGE = """ + SYNOPSIS + %prog [options] + DESCRIPTION + Upload partition files to a PartitionDB. + Called with no options, all PartitionDB objects defined in your default + domain (as specified in the "default_domain" option in the "[Persist]" + section of your boto config file) will be listed. + When called with a particular PartitionDB name (using -p option) all + Version objects of that PartitionDB object will be listed. + When called with the -p option and a particular Version name specified + (using the -v option) all Partitions in that Version object will be listed. +""" +class Client: + + def __init__(self, queue_name): + self.q = queuetools.get_queue(queue_name) + self.q.connect() + self.manager = get_manager() + self.process() + + def process(self): + m = self.q.get() + if m['item']: + v = Version(m['args']['v_id'], self.manager) + bucket_name = v.pdb.bucket_name + while m['item']: + print 'Uploading: %s' % m['item'] + p = v.add_partition(name=m['item']) + p.upload(os.path.join(m['args']['path'], m['item']), bucket_name) + self.q.delete(m) + m = self.q.get() + print 'client processing complete' + +class Server: + + def __init__(self, path, pdb_name, bucket_name=None, domain_name=None): + self.path = path + self.pdb_name = pdb_name + self.bucket_name = bucket_name + self.manager = get_manager(domain_name) + self.get_pdb() + self.serve() + + def get_pdb(self): + try: + self.pdb = PartitionDB.get(name=self.pdb_name) + except SDBPersistenceError: + self.pdb = PartitionDB(manager=self.manager, name=self.pdb_name, bucket_name=self.bucket_name) + self.pdb.save() + + def serve(self): + v = self.pdb.add_version() + args = {'path' : self.path, + 'v_id' : v.id} + l = os.listdir(self.path) + s = LQSServer('', LQSHandler, iter(l), args) + s.serve_forever() + +class Upload: + + Usage = "usage: %prog [options] command" + + Commands = {'client' : 'Start an Upload client', + 'server' : 'Start an Upload server'} + + def __init__(self): + self.parser = OptionParser(usage=self.Usage) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option('-d', '--domain-name', action='store', type='string', + help='name of the SimpleDB domain where PDB objects are stored') + self.parser.add_option('-n', '--num-processes', action='store', type='int', dest='num_processes', + help='the number of client processes launched') + self.parser.set_defaults(num_processes=2) + self.parser.add_option('-i', '--input-path', action='store', type='string', + help='the path to directory to upload') + self.parser.add_option('-p', '--pdb-name', action='store', type='string', + help='name of the PDB in which to store files (will create if necessary)') + self.parser.add_option('-b', '--bucket-name', action='store', type='string', + help='name of S3 bucket (only needed if creating new PDB)') + self.options, self.args = self.parser.parse_args() + self.prog_name = sys.argv[0] + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_server(self): + if not self.options.input_path: + self.parser.error('No path provided') + if not os.path.isdir(self.options.input_path): + self.parser.error('Invalid path (%s)' % self.options.input_path) + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + s = Server(self.options.input_path, self.options.pdb_name, + self.options.bucket_name, self.options.domain_name) + + def do_client(self): + c = Client('localhost') + + def main(self): + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) == 0: + if not self.options.input_path: + self.parser.error('No path provided') + if not os.path.isdir(self.options.input_path): + self.parser.error('Invalid path (%s)' % self.options.input_path) + if not self.options.pdb_name: + self.parser.error('No PDB name provided') + server_command = '%s -p %s -i %s' % (self.prog_name, self.options.pdb_name, self.options.input_path) + if self.options.bucket_name: + server_command += ' -b %s' % self.options.bucket_name + server_command += ' server' + client_command = '%s client' % self.prog_name + server = subprocess.Popen(server_command, shell=True) + print 'server pid: %s' % server.pid + time.sleep(5) + clients = [] + for i in range(0, self.options.num_processes): + client = subprocess.Popen(client_command, shell=True) + clients.append(client) + print 'waiting for clients to finish' + for client in clients: + client.wait() + os.kill(server.pid, signal.SIGTERM) + elif len(self.args) == 1: + self.command = self.args[0] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + else: + self.parser.error('unrecognized commands') + +if __name__ == "__main__": + upload = Upload() + upload.main() diff --git a/thirdparty/boto/mapreduce/queuetools.py b/thirdparty/boto/mapreduce/queuetools.py new file mode 100644 index 000000000..3e08a1007 --- /dev/null +++ b/thirdparty/boto/mapreduce/queuetools.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +import socket, sys +from lqs import LQSServer, LQSMessage +import boto +from boto.sqs.jsonmessage import JSONMessage + +class LQSClient: + + def __init__(self, host): + self.host = host + self.port = LQSServer.PORT + self.timeout = LQSServer.TIMEOUT + self.max_len = LQSServer.MAXSIZE + self.sock = None + + def connect(self): + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.settimeout(self.timeout) + self.sock.connect((self.host, self.port)) + + def decode(self, jsonstr): + return LQSMessage(jsonvalue=jsonstr) + + def get(self): + self.sock.send('next') + try: + jsonstr = self.sock.recv(self.max_len) + msg = LQSMessage(jsonvalue=jsonstr) + return msg + except: + print "recv from %s failed" % self.host + + def delete(self, msg): + self.sock.send('delete %s' % msg['id']) + try: + jsonstr = self.sock.recv(self.max_len) + msg = LQSMessage(jsonvalue=jsonstr) + return msg + except: + print "recv from %s failed" % self.host + + def close(self): + self.sock.close() + +class SQSClient: + + def __init__(self, queue_name): + self.queue_name = queue_name + + def connect(self): + self.queue = boto.lookup('sqs', self.queue_name) + self.queue.set_mesasge_class(JSONMessage) + + def get(self): + m = self.queue.read() + return m.get_body() + + def close(self): + pass + +def get_queue(name): + if name == 'localhost': + return LQSClient(name) + else: + return SQSClient(name) + diff --git a/thirdparty/boto/mashups/__init__.py b/thirdparty/boto/mashups/__init__.py new file mode 100644 index 000000000..449bd162a --- /dev/null +++ b/thirdparty/boto/mashups/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/mashups/interactive.py b/thirdparty/boto/mashups/interactive.py new file mode 100644 index 000000000..b80e661e5 --- /dev/null +++ b/thirdparty/boto/mashups/interactive.py @@ -0,0 +1,97 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + + +import socket +import sys + +# windows does not have termios... +try: + import termios + import tty + has_termios = True +except ImportError: + has_termios = False + + +def interactive_shell(chan): + if has_termios: + posix_shell(chan) + else: + windows_shell(chan) + + +def posix_shell(chan): + import select + + oldtty = termios.tcgetattr(sys.stdin) + try: + tty.setraw(sys.stdin.fileno()) + tty.setcbreak(sys.stdin.fileno()) + chan.settimeout(0.0) + + while True: + r, w, e = select.select([chan, sys.stdin], [], []) + if chan in r: + try: + x = chan.recv(1024) + if len(x) == 0: + print '\r\n*** EOF\r\n', + break + sys.stdout.write(x) + sys.stdout.flush() + except socket.timeout: + pass + if sys.stdin in r: + x = sys.stdin.read(1) + if len(x) == 0: + break + chan.send(x) + + finally: + termios.tcsetattr(sys.stdin, termios.TCSADRAIN, oldtty) + + +# thanks to Mike Looijmans for this code +def windows_shell(chan): + import threading + + sys.stdout.write("Line-buffered terminal emulation. Press F6 or ^Z to send EOF.\r\n\r\n") + + def writeall(sock): + while True: + data = sock.recv(256) + if not data: + sys.stdout.write('\r\n*** EOF ***\r\n\r\n') + sys.stdout.flush() + break + sys.stdout.write(data) + sys.stdout.flush() + + writer = threading.Thread(target=writeall, args=(chan,)) + writer.start() + + try: + while True: + d = sys.stdin.read(1) + if not d: + break + chan.send(d) + except EOFError: + # user hit ^Z or F6 + pass diff --git a/thirdparty/boto/mashups/iobject.py b/thirdparty/boto/mashups/iobject.py new file mode 100644 index 000000000..0937932c2 --- /dev/null +++ b/thirdparty/boto/mashups/iobject.py @@ -0,0 +1,106 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os + +def int_val_fn(v): + try: + int(v) + return True + except: + return False + +class IObject(object): + + def choose_from_list(self, item_list, search_str='', + prompt='Enter Selection'): + choice = None + while not choice: + n = 1 + choices = [] + for item in item_list: + if isinstance(item, str): + print '[%d] %s' % (n, item) + choices.append(item) + n += 1 + else: + obj, id, desc = item + if desc.find(search_str) >= 0: + print '[%d] %s - %s' % (n, id, desc) + choices.append(obj) + n += 1 + if choices: + val = raw_input('%s[1-%d]: ' % (prompt, len(choices))) + if val.startswith('/'): + search_str = val[1:] + else: + try: + int_val = int(val) + if int_val == 0: + return None + choice = choices[int_val-1] + except ValueError: + print '%s is not a valid choice' % val + except IndexError: + print '%s is not within the range[1-%d]' % (val, + len(choices)) + else: + print "No objects matched your pattern" + search_str = '' + return choice + + def get_string(self, prompt, validation_fn=None): + okay = False + while not okay: + val = raw_input('%s: ' % prompt) + if validation_fn: + okay = validation_fn(val) + if not okay: + print 'Invalid value: %s' % val + else: + okay = True + return val + + def get_filename(self, prompt): + okay = False + val = '' + while not okay: + val = raw_input('%s: %s' % (prompt, val)) + val = os.path.expanduser(val) + if os.path.isfile(val): + okay = True + elif os.path.isdir(val): + path = val + val = self.choose_from_list(os.listdir(path)) + if val: + val = os.path.join(path, val) + okay = True + else: + val = '' + else: + print 'Invalid value: %s' % val + val = '' + return val + + def get_int(self, prompt): + s = self.get_string(prompt, int_val_fn) + return int(s) + diff --git a/thirdparty/boto/mashups/order.py b/thirdparty/boto/mashups/order.py new file mode 100644 index 000000000..afe1af730 --- /dev/null +++ b/thirdparty/boto/mashups/order.py @@ -0,0 +1,199 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 order for servers +""" + +import boto +from boto.mashups.server import Server, ServerSet +from boto.mashups.iobject import IObject +from boto.pyami.config import Config +from boto.sdb.persist import get_domain, set_domain +import time, StringIO + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] + +class Item(IObject): + + def __init__(self, ec2_conn): + self.name = None + self.instance_type = None + self.quantity = 0 + self.zone = None + self.ami = None + self.groups = [] + self.key = None + self.ec2 = ec2_conn + self.config = None + + def set_userdata(self, key, value): + self.userdata[key] = value + + def get_userdata(self, key): + return self.userdata[key] + + def set_name(self, name=None): + if name: + self.name = name + else: + self.name = self.get_string('Name') + + def set_instance_type(self, instance_type=None): + if instance_type: + self.instance_type = instance_type + else: + self.instance_type = self.choose_from_list(InstanceTypes, 'Instance Type') + + def set_quantity(self, n=0): + if n > 0: + self.quantity = n + else: + self.quantity = self.get_int('Quantity') + + def set_zone(self, zone=None): + if zone: + self.zone = zone + else: + l = [(z, z.name, z.state) for z in self.ec2.get_all_zones()] + self.zone = self.choose_from_list(l, prompt='Choose Availability Zone') + + def set_ami(self, ami=None): + if ami: + self.ami = ami + else: + l = [(a, a.id, a.location) for a in self.ec2.get_all_images()] + self.ami = self.choose_from_list(l, prompt='Choose AMI') + + def add_group(self, group=None): + if group: + self.groups.append(group) + else: + l = [(s, s.name, s.description) for s in self.ec2.get_all_security_groups()] + self.groups.append(self.choose_from_list(l, prompt='Choose Security Group')) + + def set_key(self, key=None): + if key: + self.key = key + else: + l = [(k, k.name, '') for k in self.ec2.get_all_key_pairs()] + self.key = self.choose_from_list(l, prompt='Choose Keypair') + + def update_config(self): + if not self.config.has_section('Credentials'): + self.config.add_section('Credentials') + self.config.set('Credentials', 'aws_access_key_id', self.ec2.aws_access_key_id) + self.config.set('Credentials', 'aws_secret_access_key', self.ec2.aws_secret_access_key) + if not self.config.has_section('Pyami'): + self.config.add_section('Pyami') + sdb_domain = get_domain() + if sdb_domain: + self.config.set('Pyami', 'server_sdb_domain', sdb_domain) + self.config.set('Pyami', 'server_sdb_name', self.name) + + def set_config(self, config_path=None): + if not config_path: + config_path = self.get_filename('Specify Config file') + self.config = Config(path=config_path) + + def get_userdata_string(self): + s = StringIO.StringIO() + self.config.write(s) + return s.getvalue() + + def enter(self, **params): + self.name = params.get('name', self.name) + if not self.name: + self.set_name() + self.instance_type = params.get('instance_type', self.instance_type) + if not self.instance_type: + self.set_instance_type() + self.zone = params.get('zone', self.zone) + if not self.zone: + self.set_zone() + self.quantity = params.get('quantity', self.quantity) + if not self.quantity: + self.set_quantity() + self.ami = params.get('ami', self.ami) + if not self.ami: + self.set_ami() + self.groups = params.get('groups', self.groups) + if not self.groups: + self.add_group() + self.key = params.get('key', self.key) + if not self.key: + self.set_key() + self.config = params.get('config', self.config) + if not self.config: + self.set_config() + self.update_config() + +class Order(IObject): + + def __init__(self): + self.items = [] + self.reservation = None + self.ec2 = boto.connect_ec2() + + def add_item(self, **params): + item = Item(self.ec2) + item.enter(**params) + self.items.append(item) + + def display(self): + print 'This Order consists of the following items' + print + print 'QTY\tNAME\tTYPE\nAMI\t\tGroups\t\t\tKeyPair' + for item in self.items: + print '%s\t%s\t%s\t%s\t%s\t%s' % (item.quantity, item.name, item.instance_type, + item.ami.id, item.groups, item.key.name) + + def place(self, block=True): + if get_domain() == None: + print 'SDB Persistence Domain not set' + domain_name = self.get_string('Specify SDB Domain') + set_domain(domain_name) + s = ServerSet() + for item in self.items: + r = item.ami.run(min_count=1, max_count=item.quantity, + key_name=item.key.name, user_data=item.get_userdata_string(), + security_groups=item.groups, instance_type=item.instance_type, + placement=item.zone.name) + if block: + states = [i.state for i in r.instances] + if states.count('running') != len(states): + print states + time.sleep(15) + states = [i.update() for i in r.instances] + for i in r.instances: + server = Server() + server.name = item.name + server.instance_id = i.id + server.reservation = r + server.save() + s.append(server) + if len(s) == 1: + return s[0] + else: + return s + + + diff --git a/thirdparty/boto/mashups/server.py b/thirdparty/boto/mashups/server.py new file mode 100644 index 000000000..f8f7d8711 --- /dev/null +++ b/thirdparty/boto/mashups/server.py @@ -0,0 +1,345 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 server +""" +import boto, boto.utils +from boto.mashups.iobject import IObject +from boto.pyami.config import Config, BotoConfigPath +from boto.mashups.interactive import interactive_shell +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import * +import os +import StringIO + +class ServerSet(list): + + def __getattr__(self, name): + results = [] + is_callable = False + for server in self: + try: + val = getattr(server, name) + if callable(val): + is_callable = True + results.append(val) + except: + results.append(None) + if is_callable: + self.map_list = results + return self.map + return results + + def map(self, *args): + results = [] + for fn in self.map_list: + results.append(fn(*args)) + return results + +class Server(SDBObject): + + ec2 = boto.connect_ec2() + + @classmethod + def Inventory(cls): + """ + Returns a list of Server instances, one for each Server object + persisted in the db + """ + l = ServerSet() + rs = cls.find() + for server in rs: + l.append(server) + return l + + @classmethod + def Register(cls, name, instance_id, description=''): + s = cls() + s.name = name + s.instance_id = instance_id + s.description = description + s.save() + return s + + def __init__(self, id=None, manager=None): + SDBObject.__init__(self, id, manager) + self._reservation = None + self._instance = None + self._ssh_client = None + self._pkey = None + self._config = None + + name = StringProperty() + instance_id = StringProperty() + config_uri = StringProperty() + ami_id = StringProperty() + zone = StringProperty() + security_group = StringProperty() + key_name = StringProperty() + elastic_ip = StringProperty() + instance_type = StringProperty() + description = StringProperty() + + def setReadOnly(self, value): + raise AttributeError + + def getInstance(self): + if not self._instance: + if self.instance_id: + rs = self.ec2.get_all_instances([self.instance_id]) + if len(rs) > 0: + self._reservation = rs[0] + self._instance = self._reservation.instances[0] + return self._instance + + instance = property(getInstance, setReadOnly, None, 'The Instance for the server') + + def getAMI(self): + if self.instance: + return self.instance.image_id + + ami = property(getAMI, setReadOnly, None, 'The AMI for the server') + + def getStatus(self): + if self.instance: + self.instance.update() + return self.instance.state + + status = property(getStatus, setReadOnly, None, + 'The status of the server') + + def getHostname(self): + if self.instance: + return self.instance.public_dns_name + + hostname = property(getHostname, setReadOnly, None, + 'The public DNS name of the server') + + def getPrivateHostname(self): + if self.instance: + return self.instance.private_dns_name + + private_hostname = property(getPrivateHostname, setReadOnly, None, + 'The private DNS name of the server') + + def getLaunchTime(self): + if self.instance: + return self.instance.launch_time + + launch_time = property(getLaunchTime, setReadOnly, None, + 'The time the Server was started') + + def getConsoleOutput(self): + if self.instance: + return self.instance.get_console_output() + + console_output = property(getConsoleOutput, setReadOnly, None, + 'Retrieve the console output for server') + + def getGroups(self): + if self._reservation: + return self._reservation.groups + else: + return None + + groups = property(getGroups, setReadOnly, None, + 'The Security Groups controlling access to this server') + + def getConfig(self): + if not self._config: + remote_file = BotoConfigPath + local_file = '%s.ini' % self.instance.id + self.get_file(remote_file, local_file) + self._config = Config(local_file) + return self._config + + def setConfig(self, config): + local_file = '%s.ini' % self.instance.id + fp = open(local_file) + config.write(fp) + fp.close() + self.put_file(local_file, BotoConfigPath) + self._config = config + + config = property(getConfig, setConfig, None, + 'The instance data for this server') + + def stop(self): + if self.instance: + self.instance.stop() + + def start(self): + self.stop() + ec2 = boto.connect_ec2() + ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0] + groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)]) + if not self._config: + self._config = Config(fp=boto.utils.fetch_file(self.config_uri)) + if not self._config.has_section("Credentials"): + self._config.add_section("Credentials") + self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id) + self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key) + + if not self._config.has_section("Pyami"): + self._config.add_section("Pyami") + + if self.manager.domain: + self._config.set('Pyami', 'server_sdb_domain', self.manager.domain.name) + self._config.set("Pyami", 'server_sdb_name', self.name) + + cfg = StringIO.StringIO() + self._config.write(cfg) + cfg = cfg.getvalue() + r = ami.run(min_count=1, + max_count=1, + key_name=self.key_name, + security_groups = groups, + instance_type = self.instance_type, + placement = self.zone, + user_data = cfg) + i = r.instances.next() + self.instance_id = i.id + if self.elastic_ip: + ec2.associate_address(self.instance_id, self.elastic_ip) + + def reboot(self): + if self.instance: + self.instance.reboot() + + def get_ssh_client(self, key_file=None, host_key_file='~/.ssh/known_hosts', + uname='root'): + import paramiko + if not self.instance: + print 'No instance yet!' + return + if not self._ssh_client: + if not key_file: + iobject = IObject() + key_file = iobject.get_filename('Path to OpenSSH Key file') + self._pkey = paramiko.RSAKey.from_private_key_file(key_file) + self._ssh_client = paramiko.SSHClient() + self._ssh_client.load_system_host_keys() + self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) + self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self._ssh_client.connect(self.instance.public_dns_name, + username=uname, pkey=self._pkey) + return self._ssh_client + + def get_file(self, remotepath, localpath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.get(remotepath, localpath) + + def put_file(self, localpath, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.put(localpath, remotepath) + + def listdir(self, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + return sftp_client.listdir(remotepath) + + def shell(self): + ssh_client = self.get_ssh_client() + channel = ssh_client.invoke_shell() + interactive_shell(channel) + + def bundle_image(self, prefix, key_file, cert_file, size): + print 'bundling image...' + print '\tcopying cert and pk over to /mnt directory on server' + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + path, name = os.path.split(key_file) + remote_key_file = '/mnt/%s' % name + self.put_file(key_file, remote_key_file) + path, name = os.path.split(cert_file) + remote_cert_file = '/mnt/%s' % name + self.put_file(cert_file, remote_cert_file) + print '\tdeleting %s' % BotoConfigPath + # delete the metadata.ini file if it exists + try: + sftp_client.remove(BotoConfigPath) + except: + pass + command = 'ec2-bundle-vol ' + command += '-c %s -k %s ' % (remote_cert_file, remote_key_file) + command += '-u %s ' % self._reservation.owner_id + command += '-p %s ' % prefix + command += '-s %d ' % size + command += '-d /mnt ' + if self.instance.instance_type == 'm1.small' or self.instance_type == 'c1.medium': + command += '-r i386' + else: + command += '-r x86_64' + print '\t%s' % command + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def upload_bundle(self, bucket, prefix): + print 'uploading bundle...' + command = 'ec2-upload-bundle ' + command += '-m /mnt/%s.manifest.xml ' % prefix + command += '-b %s ' % bucket + command += '-a %s ' % self.ec2.aws_access_key_id + command += '-s %s ' % self.ec2.aws_secret_access_key + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def create_image(self, bucket=None, prefix=None, key_file=None, cert_file=None, size=None): + iobject = IObject() + if not bucket: + bucket = iobject.get_string('Name of S3 bucket') + if not prefix: + prefix = iobject.get_string('Prefix for AMI file') + if not key_file: + key_file = iobject.get_filename('Path to RSA private key file') + if not cert_file: + cert_file = iobject.get_filename('Path to RSA public cert file') + if not size: + size = iobject.get_int('Size (in MB) of bundled image') + self.bundle_image(prefix, key_file, cert_file, size) + self.upload_bundle(bucket, prefix) + print 'registering image...' + self.image_id = self.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) + return self.image_id + + def install_package(self, package_name): + print 'installing %s...' % package_name + command = 'yum -y install %s' % package_name + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + diff --git a/thirdparty/boto/mturk/__init__.py b/thirdparty/boto/mturk/__init__.py new file mode 100644 index 000000000..449bd162a --- /dev/null +++ b/thirdparty/boto/mturk/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/mturk/connection.py b/thirdparty/boto/mturk/connection.py new file mode 100644 index 000000000..7c759d612 --- /dev/null +++ b/thirdparty/boto/mturk/connection.py @@ -0,0 +1,197 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import xml.sax +from boto import handler +from boto.mturk.price import Price +from boto.mturk.question import QuestionForm +import boto.mturk.notification +from boto.connection import AWSQueryConnection +from boto.exception import EC2ResponseError +from boto.resultset import ResultSet + +class MTurkConnection(AWSQueryConnection): + + APIVersion = '2006-10-31' + SignatureVersion = '1' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host='mechanicalturk.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory) + + def get_account_balance(self): + params = {} + response = self.make_request('GetAccountBalance', params) + body = response.read() + if response.status == 200: + rs = ResultSet([('AvailableBalance', Price), + ('OnHoldBalance', Price)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise EC2ResponseError(response.status, response.reason, body) + + def register_hit_type(self, title, description, reward, duration, + keywords=None, approval_delay=None, qual_req=None): + """ + Register a new HIT Type + \ttitle, description are strings + \treward is a Price object + \tduration can be an integer or string + """ + params = {'Title' : title, + 'Description' : description, + 'AssignmentDurationInSeconds' : duration} + params.update(MTurkConnection.get_price_as_price(reward).get_as_params('Reward')) + response = self.make_request('RegisterHITType', params) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.HITTypeId + else: + raise EC2ResponseError(response.status, response.reason, body) + + def set_email_notification(hit_type, email): + """ + Performs a SetHITTypeNotification operation to set email notification for a specified HIT type + """ + assert type(hit_type) is str, "hit_type argument should be a string." + + params = {'HITTypeId': hit_type} + + notification_params = {'Destination': email, + 'Transport': 'Email', + 'Version': boto.mturk.notification.NOTIFICATION_VERSION } + + # Set up dict of 'Notification.1.Transport' etc. values + notification_rest_params = {} + num = 1 + for key in notification_params: + notification_rest_params['Notification.%d.%s' % (num, key)] = notification_params[key] + + # Update main params dict + params.update(notification_rest_params) + + # Execute operation + response = self.make_request('SetHITTypeNotification', params) + body = response.read() + + # From the Developer Guide: + # A successful request for the SetHITTypeNotification operation will return with no errors. The + # response will include a SetHITTypeNotificationResult element, which contains the Request (if + # the Request response group is specified). The operation returns no other data. + + if response.status == 200: # Is this the correct way to test if it worked? + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.HITTypeId + else: + raise EC2ResponseError(response.status, response.reason, body) + + + def create_hit(self, title=None, description=None, keywords=None, reward=0.00, + duration=60*60*24*7, approval_delay=None, qual_req=None, hit_type=None, + question=None, questions=None): + """ + Creates a new HIT. + Returns HITId as a string. + See: http://docs.amazonwebservices.com/AWSMechanicalTurkRequester/2006-10-31/ApiReference_CreateHITOperation.html + """ + + # handle single or multiple questions + if question is not None and questions is not None: + raise ValueError("Must specify either question (single Question instance) or questions (list), but not both") + if question is not None and questions is None: + questions = [question] + + + # Handle keywords + final_keywords = MTurkConnection.get_keywords_as_string(keywords) + + # Handle price argument + final_price = MTurkConnection.get_price_as_price(reward) + + # Set up QuestionForm data structure + qf = QuestionForm(questions=questions) + + # Handle basic arguments and set up params dict + params = {'Title': title, + 'Description' : description, + 'Keywords': final_keywords, + 'AssignmentDurationInSeconds' : duration, + 'Question': qf.get_as_xml() } + + if approval_delay is not None: + params.update({'AutoApprovalDelayInSeconds': approval_delay }) + + params.update(final_price.get_as_params('Reward')) + + # Handle optional hit_type argument + if hit_type is not None: + params.update({'HITTypeId': hit_type}) + + # Submit + response = self.make_request('CreateHIT', params) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + + return rs.HITId + #return rs # return entire ResultSet for testing purposes + else: + raise EC2ResponseError(response.status, response.reason, body) + + @staticmethod + def get_keywords_as_string(keywords): + """ + Returns a comma+space-separated string of keywords from either a list or a string + """ + if type(keywords) is list: + final_keywords = ', '.join(keywords) + elif type(keywords) is str: + final_keywords = keywords + elif keywords is None: + final_keywords = "" + else: + raise TypeError("keywords argument must be a string or a list of strings; got a %s" % type(keywords)) + return final_keywords + + @staticmethod + def get_price_as_price(reward): + """ + Returns a Price data structure from either a float or a Price + """ + if isinstance(reward, Price): + final_price = reward + else: + final_price = Price(reward) + return final_price diff --git a/thirdparty/boto/mturk/notification.py b/thirdparty/boto/mturk/notification.py new file mode 100644 index 000000000..4b3b2d970 --- /dev/null +++ b/thirdparty/boto/mturk/notification.py @@ -0,0 +1,92 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Provides NotificationMessage and Event classes, with utility methods, for +implementations of the Mechanical Turk Notification API. +""" + +import hmac +import sha +import base64 +import re + +class NotificationMessage: + + NOTIFICATION_WSDL = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurk/2006-05-05/AWSMechanicalTurkRequesterNotification.wsdl" + NOTIFICATION_VERSION = '2006-05-05' + + SERVICE_NAME = "AWSMechanicalTurkRequesterNotification" + OPERATION_NAME = "Notify" + + EVENT_PATTERN = r"Event\.(?P\d+)\.(?P\w+)" + EVENT_RE = re.compile(EVENT_PATTERN) + + def __init__(self, d): + """ + Constructor; expects parameter d to be a dict of string parameters from a REST transport notification message + """ + self.signature = d['Signature'] # vH6ZbE0NhkF/hfNyxz2OgmzXYKs= + self.timestamp = d['Timestamp'] # 2006-05-23T23:22:30Z + self.version = d['Version'] # 2006-05-05 + assert d['method'] == NotificationMessage.OPERATION_NAME, "Method should be '%s'" % NotificationMessage.OPERATION_NAME + + # Build Events + self.events = [] + events_dict = {} + if 'Event' in d: + # TurboGears surprised me by 'doing the right thing' and making { 'Event': { '1': { 'EventType': ... } } } etc. + events_dict = d['Event'] + else: + for k in d: + v = d[k] + if k.startswith('Event.'): + ed = NotificationMessage.EVENT_RE.search(k).groupdict() + n = int(ed['n']) + param = str(ed['param']) + if n not in events_dict: + events_dict[n] = {} + events_dict[n][param] = v + for n in events_dict: + self.events.append(Event(events_dict[n])) + + def verify(self, secret_key): + """ + Verifies the authenticity of a notification message. + """ + verification_input = NotificationMessage.SERVICE_NAME + NotificationMessage.OPERATION_NAME + self.timestamp + h = hmac.new(key=secret_key, digestmod=sha) + h.update(verification_input) + signature_calc = base64.b64encode(h.digest()) + return self.signature == signature_calc + +class Event: + def __init__(self, d): + self.event_type = d['EventType'] + self.event_time_str = d['EventTime'] + self.hit_type = d['HITTypeId'] + self.hit_id = d['HITId'] + self.assignment_id = d['AssignmentId'] + + #TODO: build self.event_time datetime from string self.event_time_str + + def __repr__(self): + return "" % (self.event_type, self.hit_id) diff --git a/thirdparty/boto/mturk/price.py b/thirdparty/boto/mturk/price.py new file mode 100644 index 000000000..3c88a9654 --- /dev/null +++ b/thirdparty/boto/mturk/price.py @@ -0,0 +1,48 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Price: + + def __init__(self, amount=0.0, currency_code='USD'): + self.amount = amount + self.currency_code = currency_code + self.formatted_price = '' + + def __repr__(self): + if self.formatted_price: + return self.formatted_price + else: + return str(self.amount) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Amount': + self.amount = float(value) + elif name == 'CurrencyCode': + self.currency_code = value + elif name == 'FormattedPrice': + self.formatted_price = value + + def get_as_params(self, label, ord=1): + return {'%s.%d.Amount'%(label, ord) : str(self.amount), + '%s.%d.CurrencyCode'%(label, ord) : self.currency_code} diff --git a/thirdparty/boto/mturk/question.py b/thirdparty/boto/mturk/question.py new file mode 100644 index 000000000..c54a140c4 --- /dev/null +++ b/thirdparty/boto/mturk/question.py @@ -0,0 +1,267 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Question: + + QUESTION_XML_TEMPLATE = """%s%s%s""" + + def __init__(self, identifier, content, answer_spec): #amount=0.0, currency_code='USD'): + self.identifier = identifier + self.content = content + self.answer_spec = answer_spec + + def get_as_params(self, label='Question', identifier=None): + + if identifier is None: + raise ValueError("identifier (QuestionIdentifier) is required per MTurk spec.") + + return { label : self.get_as_xml() } + + def get_as_xml(self): + ret = Question.QUESTION_XML_TEMPLATE % (self.identifier, self.content.get_as_xml(), self.answer_spec.get_as_xml()) + return ret + +class QuestionForm: + + QUESTIONFORM_SCHEMA_LOCATION = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd" + QUESTIONFORM_XML_TEMPLATE = """%s""" # % (ns, questions_xml) + + def __init__(self, questions=None): + if questions is None or type(questions) is not list: + raise ValueError("Must pass a list of Question instances to QuestionForm constructor") + else: + self.questions = questions + + def get_as_xml(self): + questions_xml = "".join([q.get_as_xml() for q in self.questions]) + return QuestionForm.QUESTIONFORM_XML_TEMPLATE % (QuestionForm.QUESTIONFORM_SCHEMA_LOCATION, questions_xml) + + #def startElement(self, name, attrs, connection): + # return None + # + #def endElement(self, name, value, connection): + # + # #if name == 'Amount': + # # self.amount = float(value) + # #elif name == 'CurrencyCode': + # # self.currency_code = value + # #elif name == 'FormattedPrice': + # # self.formatted_price = value + # + # pass # What's this method for? I don't get it. + +class QuestionContent: + + def __init__(self, title=None, text=None, bulleted_list=None, binary=None, application=None, formatted_content=None): + self.title = title + self.text = text + self.bulleted_list = bulleted_list + self.binary = binary + self.application = application + self.formatted_content = formatted_content + + def get_title_xml(self): + if self.title is None: + return '' # empty + else: + return "%s" % self.title + + def get_text_xml(self): + if self.text is None: + return '' + else: + return "%s" % self.text + + def get_bulleted_list_xml(self): + if self.bulleted_list is None: + return '' + elif type(self.bulleted_list) is list: + return "%s" % self.get_bulleted_list_items_xml() + else: + raise ValueError("QuestionContent bulleted_list argument should be a list.") + + def get_bulleted_list_items_xml(self): + ret = "" + for item in self.bulleted_list: + ret = ret + "%s" % item + return ret + + def get_binary_xml(self): + if self.binary is None: + return '' + else: + raise NotImplementedError("Binary question content is not yet supported.") + + def get_application_xml(self): + if self.application is None: + return '' + else: + raise NotImplementedError("Application question content is not yet supported.") + + def get_formatted_content_xml(self): + if self.formatted_content is None: + return '' + else: + return "" % self.formatted_content + + def get_as_xml(self): + children = self.get_title_xml() + self.get_text_xml() + self.get_bulleted_list_xml() + self.get_binary_xml() + self.get_application_xml() + self.get_formatted_content_xml() + return "%s" % children + +class AnswerSpecification: + + ANSWERSPECIFICATION_XML_TEMPLATE = """%s""" + + def __init__(self, spec): + self.spec = spec + def get_as_xml(self): + values = () # TODO + return AnswerSpecification.ANSWERSPECIFICATION_XML_TEMPLATE % self.spec.get_as_xml() + +class FreeTextAnswer: + + FREETEXTANSWER_XML_TEMPLATE = """%s%s""" # (constraints, default) + FREETEXTANSWER_CONSTRAINTS_XML_TEMPLATE = """%s%s""" # (is_numeric_xml, length_xml) + FREETEXTANSWER_LENGTH_XML_TEMPLATE = """""" # (min_length_attr, max_length_attr) + FREETEXTANSWER_ISNUMERIC_XML_TEMPLATE = """""" # (min_value_attr, max_value_attr) + FREETEXTANSWER_DEFAULTTEXT_XML_TEMPLATE = """%s""" # (default) + + def __init__(self, default=None, min_length=None, max_length=None, is_numeric=False, min_value=None, max_value=None): + self.default = default + self.min_length = min_length + self.max_length = max_length + self.is_numeric = is_numeric + self.min_value = min_value + self.max_value = max_value + + def get_as_xml(self): + is_numeric_xml = "" + if self.is_numeric: + min_value_attr = "" + max_value_attr = "" + if self.min_value: + min_value_attr = """minValue="%d" """ % self.min_value + if self.max_value: + max_value_attr = """maxValue="%d" """ % self.max_value + is_numeric_xml = FreeTextAnswer.FREETEXTANSWER_ISNUMERIC_XML_TEMPLATE % (min_value_attr, max_value_attr) + + length_xml = "" + if self.min_length or self.max_length: + min_length_attr = "" + max_length_attr = "" + if self.min_length: + min_length_attr = """minLength="%d" """ + if self.max_length: + max_length_attr = """maxLength="%d" """ + length_xml = FreeTextAnswer.FREETEXTANSWER_LENGTH_XML_TEMPLATE % (min_length_attr, max_length_attr) + + constraints_xml = "" + if is_numeric_xml != "" or length_xml != "": + constraints_xml = FreeTextAnswer.FREETEXTANSWER_CONSTRAINTS_XML_TEMPLATE % (is_numeric_xml, length_xml) + + default_xml = "" + if self.default is not None: + default_xml = FreeTextAnswer.FREETEXTANSWER_DEFAULTTEXT_XML_TEMPLATE % self.default + + return FreeTextAnswer.FREETEXTANSWER_XML_TEMPLATE % (constraints_xml, default_xml) + +class FileUploadAnswer: + FILEUPLOADANSWER_XML_TEMLPATE = """%d%d""" # (min, max) + DEFAULT_MIN_SIZE = 1024 # 1K (completely arbitrary!) + DEFAULT_MAX_SIZE = 5 * 1024 * 1024 # 5MB (completely arbitrary!) + + def __init__(self, min=None, max=None): + self.min = min + self.max = max + if self.min is None: + self.min = FileUploadAnswer.DEFAULT_MIN_SIZE + if self.max is None: + self.max = FileUploadAnswer.DEFAULT_MAX_SIZE + + def get_as_xml(self): + return FileUploadAnswer.FILEUPLOADANSWER_XML_TEMLPATE % (self.min, self.max) + +class SelectionAnswer: + """ + A class to generate SelectionAnswer XML data structures. + Does not yet implement Binary selection options. + """ + SELECTIONANSWER_XML_TEMPLATE = """%s%s""" # % (style_xml, selections_xml) + SELECTION_XML_TEMPLATE = """%s%s""" # (identifier, value_xml) + SELECTION_VALUE_XML_TEMPLATE = """<%s>%s""" # (type, value, type) + STYLE_XML_TEMPLATE = """%s""" # (style) + ACCEPTED_STYLES = ['radiobutton', 'dropdown', 'checkbox', 'list', 'combobox', 'multichooser'] + + def __init__(self, min=1, max=1, style=None, selections=None, type='text', other=False): + + if style is not None: + if style in SelectionAnswer.ACCEPTED_STYLES: + self.style_suggestion = style + else: + raise ValueError("style '%s' not recognized; should be one of %s" % (style, ', '.join(SelectionAnswer.ACCEPTED_STYLES))) + else: + self.style_suggestion = None + + if selections is None: + raise ValueError("SelectionAnswer.__init__(): selections must be a non-empty list of tuples") + else: + self.selections = selections + + self.min_selections = min + self.max_selections = max + + assert len(selections) >= self.min_selections, "# of selections is less than minimum of %d" % self.min_selections + #assert len(selections) <= self.max_selections, "# of selections exceeds maximum of %d" % self.max_selections + + self.type = type + + self.other = other + + def get_as_xml(self): + xml = "" + if self.type == 'text': + TYPE_TAG = "Text" + elif self.type == 'binary': + TYPE_TAG = "Binary" + else: + raise ValueError("illegal type: %s; must be either 'text' or 'binary'" % str(self.type)) + + # build list of elements + selections_xml = "" + for tpl in self.selections: + value_xml = SelectionAnswer.SELECTION_VALUE_XML_TEMPLATE % (TYPE_TAG, tpl[0], TYPE_TAG) + selection_xml = SelectionAnswer.SELECTION_XML_TEMPLATE % (tpl[1], value_xml) + selections_xml += selection_xml + + if self.other: + # add element + selections_xml += "" + + if self.style_suggestion is not None: + style_xml = SelectionAnswer.STYLE_XML_TEMPLATE % self.style_suggestion + else: + style_xml = "" + + ret = SelectionAnswer.SELECTIONANSWER_XML_TEMPLATE % (style_xml, selections_xml) + + # return XML + return ret + diff --git a/thirdparty/boto/pyami/__init__.py b/thirdparty/boto/pyami/__init__.py new file mode 100644 index 000000000..303dbb66c --- /dev/null +++ b/thirdparty/boto/pyami/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/thirdparty/boto/pyami/bootstrap.py b/thirdparty/boto/pyami/bootstrap.py new file mode 100644 index 000000000..59462ca97 --- /dev/null +++ b/thirdparty/boto/pyami/bootstrap.py @@ -0,0 +1,121 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import os, pwd +import boto +from boto.utils import get_instance_metadata, get_instance_userdata +from boto.pyami.config import Config, BotoConfigPath +from boto.pyami.scriptbase import ScriptBase + +class Bootstrap(ScriptBase): + """ + The Bootstrap class is instantiated and run as part of the PyAMI + instance initialization process. The methods in this class will + be run from the rc.local script of the instance and will be run + as the root user. + + The main purpose of this class is to make sure the boto distribution + on the instance is the one required. + """ + + def __init__(self): + self.working_dir = '/mnt/pyami' + self.write_metadata() + ScriptBase.__init__(self) + + def write_metadata(self): + fp = open(os.path.expanduser(BotoConfigPath), 'w') + fp.write('[Instance]\n') + inst_data = get_instance_metadata() + for key in inst_data: + fp.write('%s = %s\n' % (key, inst_data[key])) + user_data = get_instance_userdata() + fp.write('\n%s\n' % user_data) + fp.write('[Pyami]\n') + fp.write('working_dir = %s\n' % self.working_dir) + fp.close() + # This file has the AWS credentials, should we lock it down? + # os.chmod(BotoConfigPath, stat.S_IREAD | stat.S_IWRITE) + # now that we have written the file, read it into a pyami Config object + boto.config = Config() + boto.init_logging() + + def create_working_dir(self): + boto.log.info('Working directory: %s' % self.working_dir) + if not os.path.exists(self.working_dir): + os.mkdir(self.working_dir) + + def load_boto(self): + update = boto.config.get('Boto', 'boto_update', 'svn:HEAD') + if update.startswith('svn'): + if update.find(':') >= 0: + method, version = update.split(':') + version = '-r%s' % version + else: + version = '-rHEAD' + location = boto.config.get('Boto', 'boto_location', '/usr/local/boto') + self.run('svn update %s %s' % (version, location)) + else: + # first remove the symlink needed when running from subversion + self.run('rm /usr/local/lib/python2.5/site-packages/boto') + self.run('easy_install %s' % update) + + def fetch_s3_file(self, s3_file): + try: + if s3_file.startswith('s3:'): + bucket_name, key_name = s3_file[len('s3:'):].split('/') + c = boto.connect_s3() + bucket = c.get_bucket(bucket_name) + key = bucket.get_key(key_name) + boto.log.info('Fetching %s/%s' % (bucket.name, key.name)) + path = os.path.join(self.working_dir, key.name) + key.get_contents_to_filename(path) + except: + boto.log.exception('Problem Retrieving file: %s' % s3_file) + path = None + return path + + def load_packages(self): + package_str = boto.config.get('Pyami', 'packages') + if package_str: + packages = package_str.split(',') + for package in packages: + package = package.strip() + if package.startswith('s3:'): + package = self.fetch_s3_file(package) + if package: + # if the "package" is really a .py file, it doesn't have to + # be installed, just being in the working dir is enough + if not package.endswith('.py'): + self.run('easy_install -Z %s' % package, exit_on_error=False) + + def main(self): + self.create_working_dir() + self.load_boto() + self.load_packages() + self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id')) + +if __name__ == "__main__": + # because bootstrap starts before any logging configuration can be loaded from + # the boto config files, we will manually enable logging to /var/log/boto.log + boto.set_file_logger('bootstrap', '/var/log/boto.log') + bs = Bootstrap() + bs.main() diff --git a/thirdparty/boto/pyami/config.py b/thirdparty/boto/pyami/config.py new file mode 100644 index 000000000..1ca0598bd --- /dev/null +++ b/thirdparty/boto/pyami/config.py @@ -0,0 +1,141 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import StringIO, os +import ConfigParser + +BotoConfigLocations = ['/etc/boto.cfg', os.path.expanduser('~/.boto')] +BotoConfigPath = BotoConfigLocations[0] + +class Config(ConfigParser.SafeConfigParser): + + def __init__(self, path=None, fp=None): + ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami', + 'debug' : '0'}) + if path: + self.read(path) + elif fp: + self.readfp(fp) + else: + self.read(BotoConfigLocations) + + def save_option(self, path, section, option, value): + """ + Write the specified Section.Option to the config file specified by path. + Replace any previous value. If the path doesn't exist, create it. + Also add the option the the in-memory config. + """ + config = ConfigParser.SafeConfigParser() + config.read(path) + if not config.has_section(section): + config.add_section(section) + config.set(section, option, value) + fp = open(path, 'w') + config.write(fp) + fp.close() + if not self.has_section(section): + self.add_section(section) + self.set(section, option, value) + + def save_user_option(self, section, option, value): + self.save_option(os.path.expanduser(UserConfigPath), section, option, value) + + def save_system_option(self, section, option, value): + self.save_option(BotoConfigPath, section, option, value) + + def get_instance(self, name, default=None): + try: + val = self.get('Instance', name) + except: + val = default + return val + + def get_user(self, name, default=None): + try: + val = self.get('User', name) + except: + val = default + return val + + def getint_user(self, name, default=0): + try: + val = self.getint('User', name) + except: + val = default + return val + + def get_value(self, section, name, default=None): + return self.get(section, name, default) + + def get(self, section, name, default=None): + try: + val = ConfigParser.SafeConfigParser.get(self, section, name) + except: + val = default + return val + + def getint(self, section, name, default=0): + try: + val = ConfigParser.SafeConfigParser.getint(self, section, name) + except: + val = int(default) + return val + + def getfloat(self, section, name, default=0.0): + try: + val = ConfigParser.SafeConfigParser.getfloat(self, section, name) + except: + val = float(default) + return val + + def getbool(self, section, name, default=False): + if self.has_option(section, name): + val = self.get(section, name) + if val.lower() == 'true': + val = True + else: + val = False + else: + val = default + return val + + def setbool(self, section, name, value): + if value: + self.set(section, name, 'true') + else: + self.set(section, name, 'false') + + def dump(self): + s = StringIO.StringIO() + self.write(s) + print s.getvalue() + + def dump_safe(self, fp=None): + if not fp: + fp = StringIO.StringIO() + for section in self.sections(): + fp.write('[%s]\n' % section) + for option in self.options(section): + if option == 'aws_secret_access_key': + fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option) + else: + fp.write('%s = %s\n' % (option, self.get(section, option))) + diff --git a/thirdparty/boto/pyami/copybot.cfg b/thirdparty/boto/pyami/copybot.cfg new file mode 100644 index 000000000..8bf77b5ce --- /dev/null +++ b/thirdparty/boto/pyami/copybot.cfg @@ -0,0 +1,47 @@ +# +# Your AWS Credentials +# +[Credentials] +aws_access_key_id = +aws_secret_access_key = + +# +# Fill out this section if you want emails from CopyBot +# when it starts and stops +# +[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +# +# If you leave this section as is, it will automatically +# update boto from subversion upon start up. +# If you don't want that to happen, comment this out +# +[Boto] +boto_location = /usr/local/boto +boto_update = svn:HEAD + +# +# This tells the Pyami code in boto what scripts +# to run during startup +# +[Pyami] +scripts = boto.pyami.copybot.CopyBot + +# +# Source bucket and Destination Bucket, obviously. +# If exit_on_completion is false, the instance +# will keep running after the copy operation is +# complete which might be handy for debugging. +# If copy_acls is false, the ACL's will not be +# copied with the objects to the new bucket. +# +[CopyBot] +src_bucket = +dst_bucket = +exit_on_completion = true +copy_acls = true diff --git a/thirdparty/boto/pyami/copybot.py b/thirdparty/boto/pyami/copybot.py new file mode 100644 index 000000000..97c200ca6 --- /dev/null +++ b/thirdparty/boto/pyami/copybot.py @@ -0,0 +1,86 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto +from boto.pyami.scriptbase import ScriptBase +import os, StringIO + +class CopyBot(ScriptBase): + + def __init__(self): + ScriptBase.__init__(self) + self.wdir = boto.config.get('Pyami', 'working_dir') + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.wdir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + self.src_name = boto.config.get(self.name, 'src_bucket') + self.dst_name = boto.config.get(self.name, 'dst_bucket') + self.s3 = boto.connect_s3() + self.src = self.s3.lookup(self.src_name) + if not self.src: + boto.log.error('Source bucket does not exist: %s' % self.src_name) + self.dst = self.s3.lookup(self.dst_name) + if not self.dst: + self.dst = self.s3.create_bucket(self.dst_name) + + def copy_bucket_acl(self): + if boto.config.get(self.name, 'copy_acls', True): + acl = self.src.get_xml_acl() + self.dst.set_xml_acl(acl) + + def copy_key_acl(self, src, dst): + if boto.config.get(self.name, 'copy_acls', True): + acl = src.get_xml_acl() + dst.set_xml_acl(acl) + + def copy_keys(self): + boto.log.info('src=%s' % self.src.name) + boto.log.info('dst=%s' % self.dst.name) + try: + for key in self.src: + boto.log.info('copying %d bytes from key=%s' % (key.size, key.name)) + path = os.path.join(self.wdir, key.name) + key.get_contents_to_filename(path) + new_key = self.dst.new_key(key.name) + new_key.set_contents_from_filename(path) + self.copy_key_acl(key, new_key) + os.unlink(path) + except: + boto.log.exception('Error copying key: %s' % key.name) + + def copy_log(self): + key = self.dst.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + + def main(self): + fp = StringIO.StringIO() + boto.config.dump_safe(fp) + self.notify('%s (%s) Starting' % (self.name, self.instance_id), fp.getvalue()) + if self.src and self.dst: + self.copy_keys() + if self.dst: + self.copy_log() + self.notify('%s (%s) Stopping' % (self.name, self.instance_id), + 'Copy Operation Complete') + if boto.config.getbool(self.name, 'exit_on_completion', True): + ec2 = boto.connect_ec2() + ec2.terminate_instances([self.instance_id]) + diff --git a/thirdparty/boto/pyami/helloworld.py b/thirdparty/boto/pyami/helloworld.py new file mode 100644 index 000000000..680873ce1 --- /dev/null +++ b/thirdparty/boto/pyami/helloworld.py @@ -0,0 +1,28 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase + +class HelloWorld(ScriptBase): + + def main(self): + self.log('Hello World!!!') + diff --git a/thirdparty/boto/pyami/installers/__init__.py b/thirdparty/boto/pyami/installers/__init__.py new file mode 100644 index 000000000..4dcf2f426 --- /dev/null +++ b/thirdparty/boto/pyami/installers/__init__.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase +import os + +class Installer(ScriptBase): + """ + Abstract base class for installers + """ + + def add_cron(self, name, minute, hour, mday, month, wday, who, command, env=None): + """ + Add an entry to the system crontab. + """ + raise NotImplimented() + + def add_init_script(self, file): + """ + Add this file to the init.d directory + """ + + def add_env(self, key, value): + """ + Add an environemnt variable + """ + raise NotImplimented() + + def stop(self, service_name): + """ + Stop a service. + """ + raise NotImplimented() + + def start(self, service_name): + """ + Start a service. + """ + raise NotImplimented() + + def install(self): + """ + Do whatever is necessary to "install" the package. + """ + raise NotImplimented() + diff --git a/thirdparty/boto/pyami/installers/ubuntu/__init__.py b/thirdparty/boto/pyami/installers/ubuntu/__init__.py new file mode 100644 index 000000000..60ee658e3 --- /dev/null +++ b/thirdparty/boto/pyami/installers/ubuntu/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/thirdparty/boto/pyami/installers/ubuntu/apache.py b/thirdparty/boto/pyami/installers/ubuntu/apache.py new file mode 100644 index 000000000..c4ef9bf29 --- /dev/null +++ b/thirdparty/boto/pyami/installers/ubuntu/apache.py @@ -0,0 +1,36 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer + +class Apache(Installer): + """ + Install apache2, mod_python, and libapache2-svn + """ + + def install(self): + self.run('apt-get -y install apache2', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-mod-python', notify=True, exit_on_error=True) + self.run('a2enmod rewrite', notify=True, exit_on_error=True) + self.run('a2enmod ssl', notify=True, exit_on_error=True) + + def main(self): + self.install() diff --git a/thirdparty/boto/pyami/installers/ubuntu/installer.py b/thirdparty/boto/pyami/installers/ubuntu/installer.py new file mode 100644 index 000000000..160d4f391 --- /dev/null +++ b/thirdparty/boto/pyami/installers/ubuntu/installer.py @@ -0,0 +1,82 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto.pyami.installers +import os +import boto +from pwd import getpwnam + +class Installer(boto.pyami.installers.Installer): + """ + Base Installer class for Ubuntu-based AMI's + """ + def add_cron(self, name, command, minute="*", hour="*", mday="*", month="*", wday="*", who="root", env=None): + """ + Write a file to /etc/cron.d to schedule a command + env is a dict containing environment variables you want to set in the file + name will be used as the name of the file + """ + fp = open('/etc/cron.d/%s' % name, "w") + if env: + for key, value in env.items(): + fp.write('%s=%s\n' % (key, value)) + fp.write('%s %s %s %s %s %s %s\n' % (minute, hour, mday, month, wday, who, command)) + fp.close() + + def add_init_script(self, file): + """ + Add this file to the init.d directory + """ + + def add_env(self, key, value): + """ + Add an environemnt variable + For Ubuntu, the best place is /etc/environment. Values placed here do + not need to be exported. + """ + boto.log.info('Adding env variable: %s=%s' % (key, value)) + self.run('cp /etc/environment /etc/environment.orig', notify=False, exit_on_error=False) + fp = open('/etc/environment', 'a') + fp.write('\n%s=%s' % (key, value)) + fp.close() + os.environ[key] = value + + def stop(self, service_name): + self.run('/etc/init.d/%s stop' % service_name) + + def start(self, service_name): + self.run('/etc/init.d/%s start' % service_name) + + def create_user(self, user): + """ + Create a user on the local system + """ + self.run("useradd -m %s" % user) + usr = getpwnam(user) + return usr + + + def install(self): + """ + This is the only method you need to override + """ + raise NotImplimented() + diff --git a/thirdparty/boto/pyami/installers/ubuntu/mysql.py b/thirdparty/boto/pyami/installers/ubuntu/mysql.py new file mode 100644 index 000000000..1e6640880 --- /dev/null +++ b/thirdparty/boto/pyami/installers/ubuntu/mysql.py @@ -0,0 +1,76 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer +import os +import boto +from ConfigParser import SafeConfigParser +import subprocess +import time + +class MySQL(Installer): + + def install(self): + self.run('apt-get update') + self.run('apt-get -y install mysql-server', notify=True, exit_on_error=True) + + def set_root_password(self, password=None): + if not password: + password = boto.config.get('Pyami', 'mysql_root_password') + if password: + self.run('mysqladmin -u root password %s' % password) + + def change_data_dir(self): + fresh_install = False; + time.sleep(2) #trying to stop mysql immediately after installing it fails + self.stop('mysql') + if not os.path.exists('/mnt/mysql'): + self.run('mkdir /mnt/mysql') + fresh_install = True; + self.run('chown -R mysql:mysql /mnt/mysql') + fp = open('/etc/mysql/conf.d/use_mnt.cnf', 'w') + fp.write('# created by pyami\n') + fp.write('# use the /mnt volume for data\n') + fp.write('[mysqld]\n') + fp.write('datadir = /mnt/mysql\n') + fp.write('log_bin = /mnt/mysql/mysql-bin.log\n') + fp.close() + if fresh_install: + self.run('cp -pr /var/lib/mysql/* /mnt/mysql/') + self.run('cp -pr /var/log/mysql/* /mnt/mysql/') + self.start('mysql') + else: + #get the password ubuntu expects to use: + config_parser = SafeConfigParser() + config_parser.read('/etc/mysql/debian.cnf') + password = config_parser.get('client', 'password') + # start the mysql deamon, then mysql with the required grant statement piped into it: + self.start('mysql') + time.sleep(1) #time for mysql to start + grant_command = "echo \"GRANT ALL PRIVILEGES ON *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '%s' WITH GRANT OPTION;\" | mysql" % password + self.run(grant_command) + # leave mysqld running + + def main(self): + self.install() + self.set_root_password() + self.change_data_dir() + diff --git a/thirdparty/boto/pyami/installers/ubuntu/trac.py b/thirdparty/boto/pyami/installers/ubuntu/trac.py new file mode 100644 index 000000000..c97ddd29f --- /dev/null +++ b/thirdparty/boto/pyami/installers/ubuntu/trac.py @@ -0,0 +1,137 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer +import boto +import os + +class Trac(Installer): + """ + Install Trac and DAV-SVN + Sets up a Vhost pointing to [Trac]->home + Using the config parameter [Trac]->hostname + Sets up a trac environment for every directory found under [Trac]->data_dir + + [Trac] + name = My Foo Server + hostname = trac.foo.com + home = /mnt/sites/trac + data_dir = /mnt/trac + svn_dir = /mnt/subversion + server_admin = root@foo.com + sdb_auth_domain = users + # Optional + SSLCertificateFile = /mnt/ssl/foo.crt + SSLCertificateKeyFile = /mnt/ssl/foo.key + SSLCertificateChainFile = /mnt/ssl/FooCA.crt + + """ + + def install(self): + self.run('apt-get -y install trac', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True) + self.run("a2enmod ssl") + self.run("a2enmod python") + self.run("a2enmod dav_svn") + self.run("a2enmod rewrite") + + def setup_vhost(self): + domain = boto.config.get("Trac", "hostname").strip() + if domain: + cnf = open("/etc/apache2/sites-available/%s" % domain, "w") + cnf.write("NameVirtualHost *:80\n") + if boto.config.get("Trac", "SSLCertificateFile"): + cnf.write("NameVirtualHost *:443\n\n") + cnf.write("\n") + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tRewriteEngine On\n") + cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain) + cnf.write("\n\n") + + cnf.write("\n") + else: + cnf.write("\n") + + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip()) + + cnf.write("\t\n" % boto.config.get("Trac", "home").strip()) + cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n") + cnf.write("\t\tAllowOverride All\n") + cnf.write("\t\tOrder allow,deny\n") + cnf.write("\t\tallow from all\n") + cnf.write("\t\n") + + cnf.write("\t\n") + cnf.write("\t\tAuthType Basic\n") + cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name")) + cnf.write("\t\tRequire valid-user\n") + cnf.write("\t\tAuthBasicAuthoritative off\n") + cnf.write("\t\tAuthUserFile /dev/null\n") + cnf.write("\t\tPythonAuthenHandler marajo.web.authen_handler\n") + cnf.write("\t\tPythonOption SDBDomain %s\n" % boto.config.get("Trac", "sdb_auth_domain")) + cnf.write("\t\n") + + data_dir = boto.config.get("Trac", "data_dir") + for env in os.listdir(data_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tSetHandler mod_python\n") + cnf.write("\t\tPythonInterpreter main_interpreter\n") + cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n") + cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env)) + cnf.write("\t\tPythonOption TracUriRoot /trac%s\n" % env) + cnf.write("\t\n") + + svn_dir = boto.config.get("Trac", "svn_dir") + for env in os.listdir(svn_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tDAV svn\n") + cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env)) + cnf.write("\t\n") + + cnf.write("\tErrorLog /var/log/apache2/error.log\n") + cnf.write("\tLogLevel warn\n") + cnf.write("\tCustomLog /var/log/apache2/access.log combined\n") + cnf.write("\tServerSignature On\n") + SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile") + if SSLCertificateFile: + cnf.write("\tSSLEngine On\n") + cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile) + + SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile") + if SSLCertificateKeyFile: + cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile) + + SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile") + if SSLCertificateChainFile: + cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile) + cnf.write("\n") + cnf.close() + self.run("a2ensite %s" % domain) + self.run("/etc/init.d/apache2 force-reload") + + def main(self): + self.install() + self.setup_vhost() diff --git a/thirdparty/boto/pyami/launch_ami.py b/thirdparty/boto/pyami/launch_ami.py new file mode 100644 index 000000000..c49c2a33d --- /dev/null +++ b/thirdparty/boto/pyami/launch_ami.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import getopt, sys, imp, time +import boto +from boto.utils import get_instance_userdata + +usage_string = """ +SYNOPSIS + launch_ami.py -a ami_id [-b script_bucket] [-s script_name] + [-m module] [-c class_name] [-r] + [-g group] [-k key_name] [-n num_instances] + [-w] [extra_data] + Where: + ami_id - the id of the AMI you wish to launch + module - The name of the Python module containing the class you + want to run when the instance is started. If you use this + option the Python module must already be stored on the + instance in a location that is on the Python path. + script_file - The name of a local Python module that you would like + to have copied to S3 and then run on the instance + when it is started. The specified module must be + import'able (i.e. in your local Python path). It + will then be copied to the specified bucket in S3 + (see the -b option). Once the new instance(s) + start up the script will be copied from S3 and then + run locally on the instance. + class_name - The name of the class to be instantiated within the + module or script file specified. + script_bucket - the name of the bucket in which the script will be + stored + group - the name of the security group the instance will run in + key_name - the name of the keypair to use when launching the AMI + num_instances - how many instances of the AMI to launch (default 1) + input_queue_name - Name of SQS to read input messages from + output_queue_name - Name of SQS to write output messages to + extra_data - additional name-value pairs that will be passed as + userdata to the newly launched instance. These should + be of the form "name=value" + The -r option reloads the Python module to S3 without launching + another instance. This can be useful during debugging to allow + you to test a new version of your script without shutting down + your instance and starting up another one. + The -w option tells the script to run synchronously, meaning to + wait until the instance is actually up and running. It then prints + the IP address and internal and external DNS names before exiting. +""" + +def usage(): + print usage_string + sys.exit() + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w', + ['ami', 'bucket', 'class', 'group', 'help', + 'inputqueue', 'keypair', 'module', + 'numinstances', 'outputqueue', + 'reload', 'script_name', 'wait']) + except: + usage() + params = {'module_name' : None, + 'script_name' : None, + 'class_name' : None, + 'script_bucket' : None, + 'group' : 'default', + 'keypair' : None, + 'ami' : None, + 'num_instances' : 1, + 'input_queue_name' : None, + 'output_queue_name' : None} + reload = None + wait = None + for o, a in opts: + if o in ('-a', '--ami'): + params['ami'] = a + if o in ('-b', '--bucket'): + params['script_bucket'] = a + if o in ('-c', '--class'): + params['class_name'] = a + if o in ('-g', '--group'): + params['group'] = a + if o in ('-h', '--help'): + usage() + if o in ('-i', '--inputqueue'): + params['input_queue_name'] = a + if o in ('-k', '--keypair'): + params['keypair'] = a + if o in ('-m', '--module'): + params['module_name'] = a + if o in ('-n', '--num_instances'): + params['num_instances'] = int(a) + if o in ('-o', '--outputqueue'): + params['output_queue_name'] = a + if o in ('-r', '--reload'): + reload = True + if o in ('-s', '--script'): + params['script_name'] = a + if o in ('-w', '--wait'): + wait = True + + # check required fields + required = ['ami'] + for pname in required: + if not params.get(pname, None): + print '%s is required' % pname + usage() + if params['script_name']: + # first copy the desired module file to S3 bucket + if reload: + print 'Reloading module %s to S3' % params['script_name'] + else: + print 'Copying module %s to S3' % params['script_name'] + l = imp.find_module(params['script_name']) + c = boto.connect_s3() + bucket = c.get_bucket(params['script_bucket']) + key = bucket.new_key(params['script_name']+'.py') + key.set_contents_from_file(l[0]) + params['script_md5'] = key.md5 + # we have everything we need, now build userdata string + l = [] + for k, v in params.items(): + if v: + l.append('%s=%s' % (k, v)) + c = boto.connect_ec2() + l.append('aws_access_key_id=%s' % c.aws_access_key_id) + l.append('aws_secret_access_key=%s' % c.aws_secret_access_key) + for kv in args: + l.append(kv) + s = '|'.join(l) + if not reload: + rs = c.get_all_images([params['ami']]) + img = rs[0] + r = img.run(user_data=s, key_name=params['keypair'], + security_groups=[params['group']], + max_count=params.get('num_instances', 1)) + print 'AMI: %s - %s (Started)' % (params['ami'], img.location) + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + if wait: + running = False + while not running: + time.sleep(30) + [i.update() for i in r.instances] + status = [i.state for i in r.instances] + print status + if status.count('running') == len(r.instances): + running = True + for i in r.instances: + print 'Instance: %s' % i.ami_launch_index + print 'Public DNS Name: %s' % i.public_dns_name + print 'Private DNS Name: %s' % i.private_dns_name + +if __name__ == "__main__": + main() + diff --git a/thirdparty/boto/pyami/scriptbase.py b/thirdparty/boto/pyami/scriptbase.py new file mode 100644 index 000000000..89a08e0e6 --- /dev/null +++ b/thirdparty/boto/pyami/scriptbase.py @@ -0,0 +1,58 @@ +import os, sys, time, traceback +import smtplib +from boto.utils import ShellCommand, get_ts +import boto + +class ScriptBase: + + def __init__(self, config_file=None): + self.instance_id = boto.config.get('Instance', 'instance-id', 'default') + self.name = self.__class__.__name__ + self.ts = get_ts() + if config_file: + boto.config.read(config_file) + + def notify(self, subject, body=''): + to_string = boto.config.get_value('Notification', 'smtp_to', None) + if to_string: + try: + from_string = boto.config.get_value('Notification', 'smtp_from', 'boto') + msg = "From: %s\n" % from_string + msg += "To: %s\n" % to_string + msg += "Subject: %s\n\n" % subject + msg += body + smtp_host = boto.config.get_value('Notification', 'smtp_host', 'localhost') + server = smtplib.SMTP(smtp_host) + smtp_user = boto.config.get_value('Notification', 'smtp_user', '') + smtp_pass = boto.config.get_value('Notification', 'smtp_pass', '') + if smtp_user: + server.login(smtp_user, smtp_pass) + server.sendmail(from_string, to_string, msg) + server.quit() + except: + boto.log.error('notify failed') + + def mkdir(self, path): + if not os.path.isdir(path): + try: + os.mkdir(path) + except: + boto.log.error('Error creating directory: %s' % path) + + def umount(self, path): + if os.path.ismount(path): + self.run('umount %s' % path) + + def run(self, command, notify=True, exit_on_error=False): + self.last_command = ShellCommand(command) + if self.last_command.status != 0: + boto.log.error(self.last_command.output) + if notify: + self.notify('Error encountered', self.last_command.output) + if exit_on_error: + sys.exit(-1) + return self.last_command.status + + def main(self): + pass + diff --git a/thirdparty/boto/pyami/startup.py b/thirdparty/boto/pyami/startup.py new file mode 100644 index 000000000..d6f13761f --- /dev/null +++ b/thirdparty/boto/pyami/startup.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import os, sys, traceback, StringIO +import boto +from boto.utils import find_class +from boto import config +from boto.pyami.scriptbase import ScriptBase +from boto.utils import find_class + +class Startup(ScriptBase): + + def run_scripts(self): + scripts = config.get('Pyami', 'scripts') + if scripts: + for script in scripts.split(','): + script = script.strip(" ") + try: + pos = script.rfind('.') + if pos > 0: + mod_name = script[0:pos] + cls_name = script[pos+1:] + cls = find_class(mod_name, cls_name) + boto.log.info('Running Script: %s' % script) + s = cls() + s.main() + else: + boto.log.warning('Trouble parsing script: %s' % script) + except Exception, e: + boto.log.exception('Problem Running Script: %s' % script) + + def main(self): + self.run_scripts() + self.notify('Startup Completed for %s' % config.get('Instance', 'instance-id')) + +if __name__ == "__main__": + if not config.has_section('loggers'): + boto.set_file_logger('startup', '/var/log/boto.log') + sys.path.append(config.get('Pyami', 'working_dir')) + su = Startup() + su.main() diff --git a/thirdparty/boto/resultset.py b/thirdparty/boto/resultset.py new file mode 100644 index 000000000..d4481c420 --- /dev/null +++ b/thirdparty/boto/resultset.py @@ -0,0 +1,98 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ResultSet(list): + """ + The ResultSet is used to pass results back from the Amazon services + to the client. It has an ugly but workable mechanism for parsing + the XML results from AWS. Because I don't really want any dependencies + on external libraries, I'm using the standard SAX parser that comes + with Python. The good news is that it's quite fast and efficient but + it makes some things rather difficult. + + You can pass in, as the marker_elem parameter, a list of tuples. + Each tuple contains a string as the first element which represents + the XML element that the resultset needs to be on the lookout for + and a Python class as the second element of the tuple. Each time the + specified element is found in the XML, a new instance of the class + will be created and popped onto the stack. + + """ + + def __init__(self, marker_elem=None): + list.__init__(self) + if isinstance(marker_elem, list): + self.markers = marker_elem + else: + self.markers = [] + self.index = 0 + self.marker = None + self.is_truncated = False + self.next_token = None + self.status = True + + def __iter__(self): + return self + + def next(self): + if self.index == len(self): + self.index = 0 + raise StopIteration + self.index += 1 + return self[self.index-1] + + def startElement(self, name, attrs, connection): + for t in self.markers: + if name == t[0]: + obj = t[1](connection) + self.append(obj) + return obj + return None + + def to_boolean(self, value, true_value='true'): + if value == true_value: + return True + else: + return False + + def endElement(self, name, value, connection): + if name == 'IsTruncated': + self.is_truncated = self.to_boolean(value) + elif name == 'Marker': + self.marker = value + elif name == 'Prefix': + self.prefix = value + elif name == 'return': + self.status = self.to_boolean(value) + elif name == 'StatusCode': + self.status = self.to_boolean(value, 'Success') + elif name == 'ItemName': + self.append(value) + elif name == 'NextToken': + self.next_token = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + else: + setattr(self, name, value) + diff --git a/thirdparty/boto/s3/__init__.py b/thirdparty/boto/s3/__init__.py new file mode 100644 index 000000000..be2de1d51 --- /dev/null +++ b/thirdparty/boto/s3/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto + +boto.check_extensions(__name__, __path__) + +from connection import S3Connection as Connection +from key import Key +from bucket import Bucket + +__all__ = ['Connection', 'Key', 'Bucket'] diff --git a/thirdparty/boto/s3/acl.py b/thirdparty/boto/s3/acl.py new file mode 100644 index 000000000..08fb0f4fb --- /dev/null +++ b/thirdparty/boto/s3/acl.py @@ -0,0 +1,146 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.user import User +import StringIO + +CannedACLStrings = ['private', 'public-read', + 'public-read-write', 'authenticated-read'] + +class Policy: + + def __init__(self, parent=None): + self.parent = parent + self.acl = None + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + elif name == 'AccessControlList': + self.acl = ACL(self) + return self.acl + else: + return None + + def endElement(self, name, value, connection): + if name == 'Owner': + pass + elif name == 'AccessControlList': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += self.owner.to_xml() + s += self.acl.to_xml() + s += '' + return s + +class ACL: + + def __init__(self, policy=None): + self.policy = policy + self.grants = [] + + def add_grant(self, grant): + self.grants.append(grant) + + def add_email_grant(self, permission, email_address): + grant = Grant(permission=permission, type='AmazonCustomerByEmail', + email_address=email_address) + self.grants.append(grant) + + def add_user_grant(self, permission, user_id): + grant = Grant(permission=permission, type='CanonicalUser', id=user_id) + self.grants.append(grant) + + def startElement(self, name, attrs, connection): + if name == 'Grant': + self.grants.append(Grant(self)) + return self.grants[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'Grant': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + for grant in self.grants: + s += grant.to_xml() + s += '' + return s + +class Grant: + + NameSpace = 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' + + def __init__(self, permission=None, type=None, id=None, + display_name=None, uri=None, email_address=None): + self.permission = permission + self.id = id + self.display_name = display_name + self.uri = uri + self.email_address = email_address + self.type = type + + def startElement(self, name, attrs, connection): + if name == 'Grantee': + self.type = attrs['xsi:type'] + return None + + def endElement(self, name, value, connection): + if name == 'ID': + self.id = value + elif name == 'DisplayName': + self.display_name = value + elif name == 'URI': + self.uri = value + elif name == 'EmailAddress': + self.email_address = value + elif name == 'Grantee': + pass + elif name == 'Permission': + self.permission = value + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += '' % (self.NameSpace, self.type) + if self.type == 'CanonicalUser': + s += '%s' % self.id + s += '%s' % self.display_name + elif self.type == 'Group': + s += '%s' % self.uri + else: + s += '%s' % self.email_address + s += '' + s += '%s' % self.permission + s += '' + return s + + diff --git a/thirdparty/boto/s3/bucket.py b/thirdparty/boto/s3/bucket.py new file mode 100644 index 000000000..a6a2dba6c --- /dev/null +++ b/thirdparty/boto/s3/bucket.py @@ -0,0 +1,435 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto import handler +from boto.resultset import ResultSet +from boto.s3.acl import Policy, CannedACLStrings, ACL, Grant +from boto.s3.user import User +from boto.s3.key import Key +from boto.s3.prefix import Prefix +from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError +from boto.s3.bucketlistresultset import BucketListResultSet +import boto.utils +import xml.sax +import urllib + +S3Permissions = ['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'] + +class Bucket: + + BucketLoggingBody = """ + + + %s + %s + + """ + + EmptyBucketLoggingBody = """ + + """ + + LoggingGroup = 'http://acs.amazonaws.com/groups/s3/LogDelivery' + + def __init__(self, connection=None, name=None, key_class=Key): + self.name = name + self.connection = connection + self.key_class = key_class + + def __repr__(self): + return '' % self.name + + def __iter__(self): + return iter(BucketListResultSet(self)) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Name': + self.name = value + elif name == 'CreationDate': + self.creation_date = value + else: + setattr(self, name, value) + + def set_key_class(self, key_class): + """ + Set the Key class associated with this bucket. By default, this + would be the boto.s3.key.Key class but if you want to subclass that + for some reason this allows you to associate your new class with a + bucket so that when you call bucket.new_key() or when you get a listing + of keys in the bucket you will get an instances of your key class + rather than the default. + + @type key_class: class + @param key_class: A subclass of Key that can be more specific + """ + self.key_class = key_class + + def lookup(self, key_name): + """ + Deprecated: Please use get_key method. + + @type key_name: string + @param key_name: The name of the key to retrieve + + @rtype: L{Key} + @returns: A Key object from this bucket. + """ + return self.get_key(key_name) + + def get_key(self, key_name): + """ + Check to see if a particular key exists within the bucket. This + method uses a HEAD request to check for the existance of the key. + Returns: An instance of a Key object or None + + @type key_name: string + @param key_name: The name of the key to retrieve + + @rtype: L{Key} + @returns: A Key object from this bucket. + """ + response = self.connection.make_request('HEAD', self.name, key_name) + if response.status == 200: + body = response.read() + k = self.key_class(self) + k.metadata = boto.utils.get_aws_metadata(response.msg) + k.etag = response.getheader('etag') + k.content_type = response.getheader('content-type') + k.last_modified = response.getheader('last-modified') + k.size = int(response.getheader('content-length')) + k.name = key_name + return k + else: + # -- gross hack -- + # httplib gets confused with chunked responses to HEAD requests + # so I have to fake it out + response.chunked = 0 + body = response.read() + if response.status == 404: + return None + else: + raise S3ResponseError(response.status, response.reason, body) + + def list(self, prefix="", delimiter=""): + """ + List key objects within a bucket. This returns an instance of an + BucketListResultSet that automatically handles all of the result + paging, etc. from S3. You just need to keep iterating until + there are no more results. + Called with no arguments, this will return an iterator object across + all keys within the bucket. + + @type prefix: string + @param prefix: allows you to limit the listing to a particular + prefix. For example, if you call the method with prefix='/foo/' + then the iterator will only cycle through the keys that begin with + the string '/foo/'. + + @type delimiter: string + @param delimiter: can be used in conjunction with the prefix + to allow you to organize and browse your keys hierarchically. See: + http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ + for more details. + + @rtype: L{BucketListResultSet} + @return: an instance of a BucketListResultSet that handles paging, etc + """ + return BucketListResultSet(self, prefix, delimiter) + + def get_all_keys(self, headers=None, **params): + """ + Deprecated: This is better handled now by list method. + + params can be one of: prefix, marker, max-keys, delimiter + as defined in S3 Developer's Guide, however since max-keys is not + a legal variable in Python you have to pass maxkeys and this + method will munge it (Ugh!) + + @type maxkeys: int + @param maxkeys: The maximum number of keys to retrieve + + @type prefix: string + @param prefix: The prefix of the keys you want to retrieve + + @type marker: string + @param marker: The "marker" of where you are in the result set + + @type delimiter: string + @param delimiter: "If this optional, Unicode string parameter is included with your request, then keys that contain the same string between the prefix and the first occurrence of the delimiter will be rolled up into a single result element in the CommonPrefixes collection. These rolled-up keys are not returned elsewhere in the response." + + @rtype: ResultSet + @return: The result from S3 listing the keys requested + + """ + l = [] + for k,v in params.items(): + if k == 'maxkeys': + k = 'max-keys' + if isinstance(v, unicode): + v = v.encode('utf-8') + l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) + if len(l): + s = '&'.join(l) + else: + s = None + response = self.connection.make_request('GET', self.name, + headers=headers, query_args=s) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet([('Contents', self.key_class), + ('CommonPrefixes', Prefix)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise S3ResponseError(response.status, response.reason, body) + + def new_key(self, key_name=None): + """ + Creates a new key + + @type key_name: string + @param key_name: The name of the key to create + + @rtype: L{Key} or subclass + @returns: An instance of the newly created key object + """ + return self.key_class(self, key_name) + + def generate_url(self, expires_in, method='GET', headers=None): + return self.connection.generate_url(expires_in, method, self.name, headers=headers) + + def delete_key(self, key_name): + """ + Deletes a key from the bucket. + + @type key_name: string + @param key_name: The key name to delete + """ + response = self.connection.make_request('DELETE', self.name, key_name) + body = response.read() + if response.status != 204: + raise S3ResponseError(response.status, response.reason, body) + + def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=None): + """ + Create a new key in the bucket by copying another existing key. + + @type new_key_name: string + @param new_key_name: The name of the new key + + @type src_bucket_name: string + @param src_bucket_name: The name of the source bucket + + @type src_key_name: string + @param src_key_name: The name of the source key + + @type metadata: dict + @param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + @rtype: L{Key} or subclass + @returns: An instance of the newly created key object + """ + if metadata: + headers = metadata.copy() + headers['x-amz-metadata-directive'] = 'REPLACE' + else: + headers = {'x-amz-copy-source' : '%s/%s' % (src_bucket_name, src_key_name), + 'x-amz-metadata-directive' : 'COPY'} + response = self.connection.make_request('PUT', self.name, new_key_name, + headers=headers) + body = response.read() + if response.status == 200: + key = self.new_key(new_key_name) + h = handler.XmlHandler(key, self) + xml.sax.parseString(body, h) + if hasattr(key, 'Error'): + raise S3CopyError(key.Code, key.Message, body) + return key + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_canned_acl(self, acl_str, key_name=''): + assert acl_str in CannedACLStrings + response = self.connection.make_request('PUT', self.name, key_name, + headers={'x-amz-acl': acl_str}, query_args='acl') + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + + def get_xml_acl(self, key_name=''): + response = self.connection.make_request('GET', self.name, key_name, + query_args='acl') + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + return body + + def set_xml_acl(self, acl_str, key_name=''): + response = self.connection.make_request('PUT', self.name, key_name, + data=acl_str, query_args='acl') + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + + def set_acl(self, acl_or_str, key_name=''): + if isinstance(acl_or_str, Policy): + self.set_xml_acl(acl_or_str.to_xml(), key_name) + else: + self.set_canned_acl(acl_or_str, key_name) + + def get_acl(self, key_name=''): + response = self.connection.make_request('GET', self.name, key_name, + query_args='acl') + body = response.read() + if response.status == 200: + policy = Policy(self) + h = handler.XmlHandler(policy, self) + xml.sax.parseString(body, h) + return policy + else: + raise S3ResponseError(response.status, response.reason, body) + + def make_public(self, recursive=False): + self.set_canned_acl('public-read') + if recursive: + for key in self: + self.set_canned_acl('public-read', key.name) + + def add_email_grant(self, permission, email_address, recursive=False): + """ + Convenience method that provides a quick way to add an email grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + Inputs: + permission - The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + email_address - The email address associated with the AWS account your are granting + the permission to. + recursive - A boolean value to controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! + Returns: + Nothing + """ + if permission not in S3Permissions: + raise S3PermissionsError('Unknown Permission: %s' % permission) + policy = self.get_acl() + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy) + if recursive: + for key in self: + key.add_email_grant(permission, email_address) + + def add_user_grant(self, permission, user_id, recursive=False): + """ + Convenience method that provides a quick way to add a canonical user grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + @type permission: string + @param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + @type user_id: string + @param user_id: The canonical user id associated with the AWS account your are granting + the permission to. + + @type recursive: bool + @param recursive: A boolean value that controls whether the command will apply the + grant to all keys within the bucket or not. The default value is False. + By passing a True value, the call will iterate through all keys in the + bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take a long time! + """ + if permission not in S3Permissions: + raise S3PermissionsError('Unknown Permission: %s' % permission) + policy = self.get_acl() + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy) + if recursive: + for key in self: + key.add_user_grant(permission, user_id) + + def list_grants(self): + policy = self.get_acl() + return policy.acl.grants + +# def get_location(self): +# location = _XmlInterpreter( +# string_attrs={'LocationConstraint': 'location'}) +# body = self.connection.checked_request(location, self, +# 'GET', self.name, query_args='location').location +# return parse_xml_string(body, location, self) + + def enable_logging(self, target_bucket, target_prefix=''): + if isinstance(target_bucket, Bucket): + target_bucket = target_bucket.name + body = self.BucketLoggingBody % (target_bucket, target_prefix) + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging') + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def disable_logging(self): + body = self.EmptyBucketLoggingBody + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging') + body = response.read() + if response.status == 200: + return True + else: + raise S3ResponseError(response.status, response.reason, body) + + def get_logging_status(self): + response = self.connection.make_request('GET', self.name, + query_args='logging') + body = response.read() + if response.status == 200: + return body + else: + raise S3ResponseError(response.status, response.reason, body) + + def set_as_logging_target(self): + policy = self.get_acl() + g1 = Grant(permission='WRITE', type='Group', uri=self.LoggingGroup) + g2 = Grant(permission='READ_ACP', type='Group', uri=self.LoggingGroup) + policy.acl.add_grant(g1) + policy.acl.add_grant(g2) + self.set_acl(policy) diff --git a/thirdparty/boto/s3/bucketlistresultset.py b/thirdparty/boto/s3/bucketlistresultset.py new file mode 100644 index 000000000..c2e763e38 --- /dev/null +++ b/thirdparty/boto/s3/bucketlistresultset.py @@ -0,0 +1,55 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +def bucket_lister(bucket, prefix='', delimiter=''): + """ + A generator function for listing keys in a bucket. + """ + more_results = True + marker = '' + k = None + while more_results: + rs = bucket.get_all_keys(prefix=prefix, marker=marker, + delimiter=delimiter) + for k in rs: + yield k + if k: + marker = k.name + more_results= rs.is_truncated + +class BucketListResultSet: + """ + A resultset for listing keys within a bucket. Uses the bucket_lister + generator function and implements the iterator interface. This + transparently handles the results paging from S3 so even if you have + many thousands of keys within the bucket you can iterate over all + keys in a reasonably efficient manner. + """ + + def __init__(self, bucket=None, prefix='', delimiter=''): + self.bucket = bucket + self.prefix = prefix + self.delimiter = delimiter + + def __iter__(self): + return bucket_lister(self.bucket, self.prefix, self.delimiter) + + diff --git a/thirdparty/boto/s3/connection.py b/thirdparty/boto/s3/connection.py new file mode 100644 index 000000000..266276925 --- /dev/null +++ b/thirdparty/boto/s3/connection.py @@ -0,0 +1,225 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import urllib, base64 +import time +import boto.utils +from boto.connection import AWSAuthConnection +from boto import handler +from boto.s3.bucket import Bucket +from boto.s3.key import Key +from boto.resultset import ResultSet +from boto.exception import S3ResponseError, S3CreateError, BotoClientError + +def assert_case_insensitive(f): + def wrapper(*args, **kwargs): + if len(args) == 3 and not args[2].islower(): + raise BotoClientError("Bucket names cannot contain upper-case " \ + "characters when using either the sub-domain or virtual " \ + "hosting calling format.") + return f(*args, **kwargs) + return wrapper + +class _CallingFormat: + def build_url_base(self, protocol, server, bucket, key=''): + url_base = '%s://' % protocol + url_base += self.build_host(server, bucket) + url_base += self.build_path_base(bucket, key) + return url_base + + def build_host(self, server, bucket): + if bucket == '': + return server + else: + return self.get_bucket_server(server, bucket) + + def build_auth_path(self, bucket, key=''): + path = '' + if bucket != '': + path = '/' + bucket + return path + '/%s' % urllib.quote_plus(key) + + def build_path_base(self, bucket, key=''): + return '/%s' % urllib.quote_plus(key) + +class SubdomainCallingFormat(_CallingFormat): + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return '%s.%s' % (bucket, server) + +class VHostCallingFormat(_CallingFormat): + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return bucket + +class OrdinaryCallingFormat(_CallingFormat): + def get_bucket_server(self, server, bucket): + return server + + def build_path_base(self, bucket, key=''): + path_base = '/' + if bucket: + path_base += "%s/" % bucket + return path_base + urllib.quote_plus(key) + +class Location: + DEFAULT = '' + EU = 'EU' + +class S3Connection(AWSAuthConnection): + + DefaultHost = 's3.amazonaws.com' + QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0, https_connection_factory=None, + calling_format=SubdomainCallingFormat()): + self.calling_format = calling_format + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, debug=debug, + https_connection_factory=https_connection_factory) + + def __iter__(self): + return self.get_all_buckets() + + def generate_url(self, expires_in, method, bucket='', key='', + headers=None, query_auth=True): + if not headers: + headers = {} + expires = int(time.time() + expires_in) + auth_path = self.calling_format.build_auth_path(bucket, key) + canonical_str = boto.utils.canonical_string(method, auth_path, + headers, expires) + hmac_copy = self.hmac.copy() + hmac_copy.update(canonical_str) + b64_hmac = base64.encodestring(hmac_copy.digest()).strip() + encoded_canonical = urllib.quote_plus(b64_hmac) + path = self.calling_format.build_path_base(bucket, key) + if query_auth: + query_part = '?' + self.QueryString % (encoded_canonical, expires, + self.aws_access_key_id) + else: + query_part = '' + return self.calling_format.build_url_base(self.protocol, + self.server_name, bucket, key) + query_part + + def get_all_buckets(self): + response = self.make_request('GET') + body = response.read() + if response.status > 300: + raise S3ResponseError(response.status, response.reason, body) + rs = ResultSet([('Bucket', Bucket)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def get_canonical_user_id(self): + """ + Convenience method that returns the "CanonicalUserID" of the user who's credentials + are associated with the connection. The only way to get this value is to do a GET + request on the service which returns all buckets associated with the account. As part + of that response, the canonical userid is returned. This method simply does all of + that and then returns just the user id. + Returns: + A string containing the canonical user id. + """ + rs = self.get_all_buckets() + return rs.ID + + def get_bucket(self, bucket_name): + bucket = Bucket(self, bucket_name) + rs = bucket.get_all_keys(None, maxkeys=0) + return bucket + + def lookup(self, bucket_name): + try: + bucket = self.get_bucket(bucket_name) + except: + bucket = None + return bucket + + def create_bucket(self, bucket_name, headers=None, location=Location.DEFAULT, policy=None): + """ + Creates a new located bucket. By default it's in the USA. You can pass + Location.EU to create an European bucket. + + @type bucket_name: string + @param bucket_name: The name of the new bucket + + @type headers: dict + @param headers: Additional headers to pass along with the request to AWS. + + @type location: L{Location} + @param location: The location of the new bucket + + @type policy: L{CannedACLString} + @param policy: A canned ACL policy that will be applied to the new key in S3. + + """ + if policy: + if headers: + headers['x-amz-acl'] = policy + else: + headers = {'x-amz-acl' : policy} + if location == Location.DEFAULT: + data = '' + else: + data = '' + \ + location + '' + response = self.make_request('PUT', bucket_name, headers=headers, + data=data) + body = response.read() + if response.status == 409: + raise S3CreateError(response.status, response.reason, body) + if response.status == 200: + return Bucket(self, bucket_name) + else: + raise S3ResponseError(response.status, response.reason, body) + + def delete_bucket(self, bucket): + response = self.make_request('DELETE', bucket) + body = response.read() + if response.status != 204: + raise S3ResponseError(response.status, response.reason, body) + + def make_request(self, method, bucket='', key='', headers=None, data='', + query_args=None, sender=None): + if isinstance(bucket, Bucket): + bucket = bucket.name + if isinstance(key, Key): + key = key.name + path = self.calling_format.build_path_base(bucket, key) + auth_path = self.calling_format.build_auth_path(bucket, key) + host = self.calling_format.build_host(self.server, bucket) + if query_args: + path += '?' + query_args + auth_path += '?' + query_args + return AWSAuthConnection.make_request(self, method, path, headers, + data, host, auth_path, sender) + +# def checked_request(self, method, bucket='', key='', headers=None, data='', +# query_args=None, sender=None, good_status=200): +# response = self.make_request(method, bucket, key, headers, data, +# query_args, sender) +# return check_s3_response(response, good_status) diff --git a/thirdparty/boto/s3/key.py b/thirdparty/boto/s3/key.py new file mode 100644 index 000000000..3ea0db7f0 --- /dev/null +++ b/thirdparty/boto/s3/key.py @@ -0,0 +1,728 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import socket +import mimetypes +import md5 +import os +import rfc822 +import StringIO +import time +import base64 +import boto +import boto.utils +from boto.exception import S3ResponseError, S3DataError, BotoClientError +from boto.s3.user import User +from boto import UserAgent, config + +class Key: + + DefaultContentType = 'application/octet-stream' + + BufferSize = 8192 + + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + self.metadata = {} + self.content_type = self.DefaultContentType + self.filename = None + self.etag = None + self.last_modified = None + self.owner = None + self.storage_class = None + self.md5 = None + self.base64md5 = None + self.path = None + self.resp = None + self.mode = None + self.size = None + + def __repr__(self): + if self.bucket: + return '' % (self.bucket.name, self.name) + else: + return '' % self.name + + def __getattr__(self, name): + if name == 'key': + return self.name + else: + raise AttributeError + + def __setattr__(self, name, value): + if name == 'key': + self.__dict__['name'] = value + else: + self.__dict__[name] = value + + def __iter__(self): + return self + + def open_read(self, headers=None, query_args=None): + """ + Open this key for reading + + @type headers: dict + @param headers: Headers to pass in the web request + + @type query_args: string + @param query_args: Arguments to pass in the query string (ie, 'torrent') + """ + if self.resp == None: + self.mode = 'r' + + self.resp = self.bucket.connection.make_request('GET', self.bucket.name, self.name, headers, query_args=query_args) + if self.resp.status < 199 or self.resp.status > 299: + raise S3ResponseError(self.resp.status, self.resp.reason) + response_headers = self.resp.msg + self.metadata = boto.utils.get_aws_metadata(response_headers) + for name,value in response_headers.items(): + if name.lower() == 'content-length': + self.size = int(value) + elif name.lower() == 'etag': + self.etag = value + elif name.lower() == 'content-type': + self.content_type = value + elif name.lower() == 'last-modified': + self.last_modified = value + + def open_write(self, headers=None): + """ + Open this key for writing. + Not yet implemented + + @type headers: dict + @param headers: Headers to pass in the write request + """ + raise BotoClientError('Not Implemented') + + def open(self, mode='r', headers=None, query_args=None): + if mode == 'r': + self.mode = 'r' + self.open_read(headers=headers, query_args=query_args) + elif mode == 'w': + self.mode = 'w' + self.open_write(headers=headers) + else: + raise BotoClientError('Invalid mode: %s' % mode) + + def close(self): + if self.resp: + self.resp.read() + self.resp = None + self.mode = None + + def next(self): + """ + By providing a next method, the key object supports use as an iterator. + For example, you can now say: + + for bytes in key: + write bytes to a file or whatever + + All of the HTTP connection stuff is handled for you. + """ + self.open_read() + data = self.resp.read(self.BufferSize) + if not data: + self.close() + raise StopIteration + return data + + def read(self, size=0): + if size == 0: + size = self.BufferSize + self.open_read() + data = self.resp.read(size) + if not data: + self.close() + return data + + def copy(self, dst_bucket, dst_key, metadata=None): + """ + Copy this Key to another bucket. + + @type dst_bucket: string + @param dst_bucket: The name of the destination bucket + + @type dst_key: string + @param dst_key: The name of the destinatino key + + @type metadata: dict + @param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + @rtype: L{Key} or subclass + @returns: An instance of the newly created key object + """ + dst_bucket = self.bucket.connection.lookup(dst_bucket) + return dst_bucket.copy_key(dst_key, self.bucket.name, self.name, metadata) + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + else: + return None + + def endElement(self, name, value, connection): + if name == 'Key': + self.name = value.encode('utf-8') + elif name == 'ETag': + self.etag = value + elif name == 'LastModified': + self.last_modified = value + elif name == 'Size': + self.size = int(value) + elif name == 'StorageClass': + self.storage_class = value + elif name == 'Owner': + pass + else: + setattr(self, name, value) + + def exists(self): + """ + Returns True if the key exists + + @rtype: bool + @return: Whether the key exists on S3 + """ + return bool(self.bucket.lookup(self.name)) + + def delete(self): + """ + Delete this key from S3 + """ + return self.bucket.delete_key(self.name) + + def get_metadata(self, name): + return self.metadata.get(name) + + def set_metadata(self, name, value): + self.metadata[name] = value + + def update_metadata(self, d): + self.metadata.update(d) + + # convenience methods for setting/getting ACL + def set_acl(self, acl_str): + if self.bucket != None: + self.bucket.set_acl(acl_str, self.name) + + def get_acl(self): + if self.bucket != None: + return self.bucket.get_acl(self.name) + + def get_xml_acl(self): + if self.bucket != None: + return self.bucket.get_xml_acl(self.name) + + def set_xml_acl(self, acl_str): + if self.bucket != None: + return self.bucket.set_xml_acl(acl_str, self.name) + + def make_public(self): + response = self.bucket.connection.make_request('PUT', self.bucket.name, self.name, + headers={'x-amz-acl': 'public-read'}, query_args='acl') + body = response.read() + if response.status != 200: + raise S3ResponseError(response.status, response.reason, body) + + def generate_url(self, expires_in, method='GET', headers=None, query_auth=True): + """ + Generate a URL to access this key. + + @type expires_in: int + @param expires_in: How long the url is valid for + + @type method: string + @param method: The method to use for retrieving the file (default is GET) + + @type headers: dict + @param headers: Any headers to pass along in the request + + @type query_auth: bool + @param query_auth: + + @rtype: string + @return: The URL to access the key + """ + return self.bucket.connection.generate_url(expires_in, method, + self.bucket.name, self.name, + headers, query_auth) + + def send_file(self, fp, headers=None, cb=None, num_cb=10): + """ + Upload a file to a key into a bucket on S3. + + @type fp: file + @param fp: The file pointer to upload + + @type headers: dict + @param headers: The headers to pass along with the PUT request + + @type cb: function + @param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + """ + def sender(http_conn, method, path, data, headers): + http_conn.putrequest('PUT', path) + for key in headers: + http_conn.putheader(key, headers[key]) + http_conn.endheaders() + fp.seek(0) + save_debug = self.bucket.connection.debug + self.bucket.connection.debug = 0 + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + l = fp.read(self.BufferSize) + while len(l) > 0: + http_conn.send(l) + if cb: + total_bytes += len(l) + i += 1 + if i == cb_count: + cb(total_bytes, self.size) + i = 0 + l = fp.read(self.BufferSize) + if cb: + cb(total_bytes, self.size) + response = http_conn.getresponse() + body = response.read() + fp.seek(0) + self.bucket.connection.debug = save_debug + if response.status == 500 or response.status == 503 or \ + response.getheader('location'): + # we'll try again + return response + elif response.status >= 200 and response.status <= 299: + self.etag = response.getheader('etag') + if self.etag != '"%s"' % self.md5: + raise S3DataError('ETag from S3 did not match computed MD5') + return response + else: + raise S3ResponseError(response.status, response.reason, body) + + if not headers: + headers = {} + else: + headers = headers.copy() + headers['User-Agent'] = UserAgent + headers['Content-MD5'] = self.base64md5 + if headers.has_key('Content-Type'): + self.content_type = headers['Content-Type'] + elif self.path: + self.content_type = mimetypes.guess_type(self.path)[0] + if self.content_type == None: + self.content_type = self.DefaultContentType + headers['Content-Type'] = self.content_type + else: + headers['Content-Type'] = self.content_type + headers['Content-Length'] = self.size + headers['Expect'] = '100-Continue' + headers = boto.utils.merge_meta(headers, self.metadata) + return self.bucket.connection.make_request('PUT', self.bucket.name, + self.name, headers, sender=sender) + + def _compute_md5(self, fp): + """ + @type fp: file + @param fp: File pointer to the file to MD5 hash + + @rtype: string + @return: MD5 Hash of the file in fp + """ + m = md5.new() + s = fp.read(self.BufferSize) + while s: + m.update(s) + s = fp.read(self.BufferSize) + self.md5 = m.hexdigest() + self.base64md5 = base64.encodestring(m.digest()) + if self.base64md5[-1] == '\n': + self.base64md5 = self.base64md5[0:-1] + self.size = fp.tell() + fp.seek(0) + + def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, policy=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file pointed to by 'fp' as the + contents. + + @type fp: file + @param fp: the file whose contents to upload + + @type headers: dict + @param headers: additional HTTP headers that will be sent with the PUT request. + + @type replace: bool + @param replace: If this parameter is False, the method + will first check to see if an object exists in the + bucket with the same key. If it does, it won't + overwrite it. The default value is True which will + overwrite the object. + + @type cb: function + @param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + @type policy: L{CannedACLString} + @param policy: A canned ACL policy that will be applied to the new key in S3. + + """ + if policy: + if headers: + headers['x-amz-acl'] = policy + else: + headers = {'x-amz-acl' : policy} + if hasattr(fp, 'name'): + self.path = fp.name + if self.bucket != None: + self._compute_md5(fp) + if self.name == None: + self.name = self.md5 + if not replace: + k = self.bucket.lookup(self.name) + if k: + return + self.send_file(fp, headers, cb, num_cb) + + def set_contents_from_filename(self, filename, headers=None, replace=True, cb=None, num_cb=10, policy=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file named by 'filename'. + See set_contents_from_file method for details about the + parameters. + + @type filename: string + @param filename: The name of the file that you want to put onto S3 + + @type headers: dict + @param headers: Additional headers to pass along with the request to AWS. + + @type replace: bool + @param replace: If True, replaces the contents of the file if it already exists. + + @type cb: function + @param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + @type policy: L{CannedACLString} + @param policy: A canned ACL policy that will be applied to the new key in S3. + + """ + fp = open(filename, 'rb') + self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) + fp.close() + + def set_contents_from_string(self, s, headers=None, replace=True, cb=None, num_cb=10, policy=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the string 's' as the contents. + See set_contents_from_file method for details about the + parameters. + + @type headers: dict + @param headers: Additional headers to pass along with the request to AWS. + + @type replace: bool + @param replace: If True, replaces the contents of the file if it already exists. + + @type cb: function + @param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + @type policy: L{CannedACLString} + @param policy: A canned ACL policy that will be applied to the new key in S3. + + """ + fp = StringIO.StringIO(s) + self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy) + fp.close() + + def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieves a file from an S3 Key + + @type fp: file + @param fp: File pointer to put the data into + + @type headers: string + @param: headers to send when retrieving the files + + @type cb: function + @param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + @type torrent: bool + @param torrent: Flag for whether to get a torrent for the file + """ + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + save_debug = self.bucket.connection.debug + if self.bucket.connection.debug == 1: + self.bucket.connection.debug = 0 + + if torrent: torrent = "torrent" + self.open('r', headers, query_args=torrent) + for bytes in self: + fp.write(bytes) + if cb: + total_bytes += len(bytes) + i += 1 + if i == cb_count: + cb(total_bytes, self.size) + i = 0 + if cb: + cb(total_bytes, self.size) + self.close() + self.bucket.connection.debug = save_debug + + def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10): + """ + Get a torrent file (see to get_file) + + @type fp: file + @param fp: The file pointer of where to put the torrent + + @type headers: dict + @param headers: Headers to be passed + + @type cb: function + @param cb: Callback function to call on retrieved data + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + """ + return self.get_file(fp, headers, cb, num_cb, torrent=True) + + def get_contents_to_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Write the contents of the object to the file pointed + to by 'fp'. + + @type fp: File -like object + @param fp: + + @type headers: dict + @param headers: additional HTTP headers that will be sent with the GET request. + + @type cb: function + @param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + @type torrent: bool + @param torrent: If True, returns the contents of a torrent file as a string. + + """ + if self.bucket != None: + self.get_file(fp, headers, cb, num_cb, torrent=torrent) + + def get_contents_to_filename(self, filename, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Store contents of the object to a file named by 'filename'. + See get_contents_to_file method for details about the + parameters. + + @type filename: string + @param filename: The filename of where to put the file contents + + @type headers: dict + @param headers: Any additional headers to send in the request + + @type cb: function + @param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + @type torrent: bool + @param torrent: If True, returns the contents of a torrent file as a string. + + """ + fp = open(filename, 'wb') + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) + fp.close() + # if last_modified date was sent from s3, try to set file's timestamp + if self.last_modified != None: + try: + modified_tuple = rfc822.parsedate_tz(self.last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + os.utime(fp.name, (modified_stamp, modified_stamp)) + except Exception, e: pass + + def get_contents_as_string(self, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Return the contents of the object as a string. + See get_contents_to_file method for details about the + parameters. + + @type headers: dict + @param headers: Any additional headers to send in the request + + @type cb: function + @param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + + @type cb: int + @param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + @type torrent: bool + @param torrent: If True, returns the contents of a torrent file as a string. + + @rtype: string + @returns: The contents of the file as a string + """ + fp = StringIO.StringIO() + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent) + return fp.getvalue() + + def add_email_grant(self, permission, email_address): + """ + Convenience method that provides a quick way to add an email grant to a key. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + @type permission: string + @param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + @type email_address: string + @param email_address: The email address associated with the AWS account your are granting + the permission to. + """ + policy = self.get_acl() + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy) + + def add_user_grant(self, permission, user_id): + """ + Convenience method that provides a quick way to add a canonical user grant to a key. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. + + @type permission: string + @param permission: The permission being granted. Should be one of: + READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL + See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html + for more details on permissions. + + @type user_id: string + @param user_id: The canonical user id associated with the AWS account your are granting + the permission to. + """ + policy = self.get_acl() + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy) diff --git a/thirdparty/boto/s3/prefix.py b/thirdparty/boto/s3/prefix.py new file mode 100644 index 000000000..fc0f26ab5 --- /dev/null +++ b/thirdparty/boto/s3/prefix.py @@ -0,0 +1,35 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Prefix: + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Prefix': + self.name = value + else: + setattr(self, name, value) + diff --git a/thirdparty/boto/s3/user.py b/thirdparty/boto/s3/user.py new file mode 100644 index 000000000..f45f03813 --- /dev/null +++ b/thirdparty/boto/s3/user.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class User: + def __init__(self, parent=None, id='', display_name=''): + if parent: + parent.owner = self + self.type = None + self.id = id + self.display_name = display_name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DisplayName': + self.display_name = value + elif name == 'ID': + self.id = value + else: + setattr(self, name, value) + + def to_xml(self, element_name='Owner'): + if self.type: + s = '<%s xsi:type="%s">' % (element_name, self.type) + else: + s = '<%s>' % element_name + s += '%s' % self.id + s += '%s' % self.display_name + s += '' % element_name + return s diff --git a/thirdparty/boto/sdb/__init__.py b/thirdparty/boto/sdb/__init__.py new file mode 100644 index 000000000..449bd162a --- /dev/null +++ b/thirdparty/boto/sdb/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/sdb/connection.py b/thirdparty/boto/sdb/connection.py new file mode 100644 index 000000000..c24d2109e --- /dev/null +++ b/thirdparty/boto/sdb/connection.py @@ -0,0 +1,242 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import urllib +import xml.sax +import threading +import boto +from boto import handler +from boto.connection import AWSQueryConnection +from boto.sdb.domain import Domain +from boto.sdb.item import Item +from boto.exception import SDBResponseError +from boto.resultset import ResultSet + +class ItemThread(threading.Thread): + + def __init__(self, name, domain_name, item_names): + threading.Thread.__init__(self, name=name) + print 'starting %s with %d items' % (name, len(item_names)) + self.domain_name = domain_name + self.conn = SDBConnection() + self.item_names = item_names + self.items = [] + + def run(self): + for item_name in self.item_names: + item = self.conn.get_attributes(self.domain_name, item_name) + self.items.append(item) + +class SDBConnection(AWSQueryConnection): + + APIVersion = '2007-11-07' + SignatureVersion = '1' + ResponseError = SDBResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host='sdb.amazonaws.com', debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory) + self.box_usage = 0.0 + + def build_name_value_list(self, params, attributes, replace=False): + keys = attributes.keys() + keys.sort() + i = 1 + for key in keys: + value = attributes[key] + if isinstance(value, list): + for v in value: + params['Attribute.%d.Name'%i] = key + params['Attribute.%d.Value'%i] = v + i += 1 + else: + params['Attribute.%d.Name'%i] = key + params['Attribute.%d.Value'%i] = value + if replace: + params['Attribute.%d.Replace'%i] = 'true' + i += 1 + + def build_name_list(self, params, attribute_names): + i = 1 + attribute_names.sort() + for name in attribute_names: + params['Attribute.%d.Name'%i] = name + i += 1 + + def get_usage(self): + return self.box_usage + + def print_usage(self): + print 'Total Usage: %f compute seconds' % self.box_usage + cost = self.box_usage * 0.14 + print 'Approximate Cost: $%f' % cost + + def get_domain(self, domain_name, validate=True): + """ + Returns a Domain object for a given domain_name. + If the validate parameter is True, the domain_name is validated + by performing a query (returning a max of 1 item) against the domain. + """ + domain = Domain(self, domain_name) + if validate: + self.query(domain, '', max_items=1) + return domain + + def lookup(self, domain_name): + try: + domain = self.get_domain(domain_name) + except: + domain = None + return domain + + def get_all_domains(self, max_domains=None, next_token=None): + params = {} + if max_domains: + params['MaxNumberOfDomains'] = max_domains + if next_token: + params['NextToken'] = next_token + return self.get_list('ListDomains', params, [('DomainName', Domain)]) + + def create_domain(self, domain_name): + params = {'DomainName':domain_name} + d = self.get_object('CreateDomain', params, Domain) + d.name = domain_name + return d + + def get_domain_and_name(self, domain_or_name): + if (isinstance(domain_or_name, Domain)): + return (domain_or_name, domain_or_name.name) + else: + return (self.get_domain(domain_or_name), domain_or_name) + + def delete_domain(self, domain_or_name): + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name} + return self.get_status('DeleteDomain', params) + + def put_attributes(self, domain_or_name, item_name, attributes, replace=True): + """ + Store attributes for a given item in a domain. + Parameters: + domain__or_name - either a domain object or the name of a domain in SimpleDB + item_name - the name of the SDB item the attributes will be + associated with + attributes - a dict containing the name/value pairs to store + as attributes + replace - a boolean value that determines whether the attribute + values passed in will replace any existing values or will + be added as additional values. Defaults to True. + Returns: + Boolean True or raises an exception + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + self.build_name_value_list(params, attributes, replace) + return self.get_status('PutAttributes', params) + + def get_attributes(self, domain_or_name, item_name, attribute_name=None, item=None): + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + if attribute_name: + params['AttributeName'] = attribute_name + response = self.make_request('GetAttributes', params) + body = response.read() + if response.status == 200: + if item == None: + item = Item(domain, item_name) + h = handler.XmlHandler(item, self) + xml.sax.parseString(body, h) + return item + else: + raise SDBResponseError(response.status, response.reason, body) + + def delete_attributes(self, domain_or_name, item_name, attr_names=None): + """ + Delete attributes from a given item in a domain. + Parameters: + domain__or_name - either a domain object or the name of a domain in SimpleDB + item_name - the name of the SDB item the attributes will be + removed from + attributes - either a list containing attribute names which will cause + all values associated with that attribute name to be deleted or + a dict containing the attribute names and keys and list of values + to delete as the value + Returns: + Boolean True or raises an exception + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'ItemName' : item_name} + if attr_names: + if isinstance(attr_names, list): + self.build_name_list(params, attr_names) + elif isinstance(attr_names, dict): + self.build_name_value_list(params, attr_names) + return self.get_status('DeleteAttributes', params) + + def query(self, domain_or_name, query='', max_items=None, next_token=None): + """ + Returns a list of item names within domain_name that match the query. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'QueryExpression' : query} + if max_items: + params['MaxNumberOfItems'] = max_items + if next_token: + params['NextToken'] = next_token + return self.get_object('Query', params, ResultSet) + + def threaded_query(self, domain_or_name, query='', max_items=None, next_token=None, num_threads=6): + """ + Returns a list of fully populated items that match the query provided. + + The name/value pairs for all of the matching item names are retrieved in a number of separate + threads (specified by num_threads) to achieve maximum throughput. + The ResultSet that is returned has an attribute called next_token that can be used + to retrieve additional results for the same query. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + if max_items and num_threads > max_items: + num_threads = max_items + rs = self.query(domain_or_name, query, max_items, next_token) + threads = [] + n = len(rs) / num_threads + for i in range(0, num_threads): + if i+1 == num_threads: + thread = ItemThread('Thread-%d' % i, domain_name, rs[n*i:]) + else: + thread = ItemThread('Thread-%d' % i, domain_name, rs[n*i:n*(i+1)]) + threads.append(thread) + thread.start() + del rs[0:] + for thread in threads: + thread.join() + for item in thread.items: + rs.append(item) + return rs + diff --git a/thirdparty/boto/sdb/domain.py b/thirdparty/boto/sdb/domain.py new file mode 100644 index 000000000..9f779916a --- /dev/null +++ b/thirdparty/boto/sdb/domain.py @@ -0,0 +1,74 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SDB Domain +""" +from boto.sdb.queryresultset import QueryResultSet +from boto.sdb.item import Item + +class Domain: + + def __init__(self, connection=None, name=None): + self.connection = connection + self.name = name + + def __repr__(self): + return 'Domain:%s' % self.name + + def __iter__(self): + return iter(QueryResultSet(self)) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DomainName': + self.name = value + else: + setattr(self, name, value) + + def put_attributes(self, item_name, attributes, replace=True): + return self.connection.put_attributes(self, item_name, attributes, replace) + + def get_attributes(self, item_name, attribute_name=None, item=None): + return self.connection.get_attributes(self, item_name, attribute_name, item) + + def delete_attributes(self, item_name, attributes=None): + return self.connection.delete_attributes(self, item_name, attributes) + + def query(self, query='', max_items=None): + return iter(QueryResultSet(self, query, max_items)) + + def get_item(self, item_name): + item = self.get_attributes(item_name) + if item: + item.domain = self + return item + else: + return None + + def new_item(self, item_name): + return Item(self, item_name) + + def delete_item(self, item): + self.delete_attributes(item.name) + diff --git a/thirdparty/boto/sdb/item.py b/thirdparty/boto/sdb/item.py new file mode 100644 index 000000000..59bd9b957 --- /dev/null +++ b/thirdparty/boto/sdb/item.py @@ -0,0 +1,111 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SDB Item +""" + +from UserDict import DictMixin + +class Item(DictMixin): + + def __init__(self, domain, name='', active=True): + self.domain = domain + self.name = name + self._dict = None + self.active = active + self.request_id = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'ItemName': + self.name = value + elif name == 'Name': + self.last_key = value + elif name == 'Value': + if self._dict == None: + self._dict = {} + if self._dict.has_key(self.last_key): + if not isinstance(self._dict[self.last_key], list): + self._dict[self.last_key] = [self._dict[self.last_key]] + self._dict[self.last_key].append(value) + else: + self._dict[self.last_key] = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + elif name == 'RequestId': + self.request_id = value + else: + setattr(self, name, value) + + def load(self): + if self._dict == None: + self._dict = {} + self.domain.get_attributes(self.name, item=self) + + def save(self, replace=True): + self.domain.put_attributes(self.name, self, replace) + + def __getitem__(self, key): + if self._dict == None: + self.load() + return self._dict[key] + + def __setitem__(self, key, value): + if self._dict == None: + self.load() + if self.active: + self.domain.put_attributes(self.name, {key : value}) + self._dict[key] = value + + def __delitem__(self, key): + if self._dict == None: + self.load() + if self.active: + self.domain.delete_attributes(self.name, [key]) + del self._dict[key] + + def keys(self): + if self._dict == None: + self.load() + return self._dict.keys() + + def add_value(self, key, value): + if self.has_key(key): + if self.active: + self.domain.put_attributes(self.name, {key : value}, replace=False) + if not isinstance(self._dict[key], list): + self._dict[key] = [self._dict[key]] + self._dict[key].append(value) + else: + self[key] = value + + def delete(self): + self.domain.delete_item(self) + + + + diff --git a/thirdparty/boto/sdb/persist/__init__.py b/thirdparty/boto/sdb/persist/__init__.py new file mode 100644 index 000000000..17d588e51 --- /dev/null +++ b/thirdparty/boto/sdb/persist/__init__.py @@ -0,0 +1,83 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.utils import find_class + +class Manager(object): + + DefaultDomainName = boto.config.get('Persist', 'default_domain', None) + + def __init__(self, domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + self.domain_name = domain_name + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + self.domain = None + self.sdb = None + self.s3 = None + if not self.domain_name: + self.domain_name = self.DefaultDomainName + if self.domain_name: + boto.log.info('No SimpleDB domain set, using default_domain: %s' % self.domain_name) + else: + boto.log.warning('No SimpleDB domain set, persistance is disabled') + if self.domain_name: + self.sdb = boto.connect_sdb(aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + debug=debug) + self.domain = self.sdb.lookup(self.domain_name) + if not self.domain: + self.domain = self.sdb.create_domain(self.domain_name) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) + return self.s3 + +def get_manager(domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + return Manager(domain_name, aws_access_key_id, aws_secret_access_key, debug=debug) + +def set_domain(domain_name): + Manager.DefaultDomainName = domain_name + +def get_domain(): + return Manager.DefaultDomainName + +def revive_object_from_id(id, manager): + if not manager.domain: + return None + attrs = manager.domain.get_attributes(id, ['__module__', '__type__', '__lineage__']) + try: + cls = find_class(attrs['__module__'], attrs['__type__']) + return cls(id, manager=manager) + except ImportError: + return None + +def object_lister(cls, query_lister, manager): + for item in query_lister: + if cls: + yield cls(item.name) + else: + o = revive_object_from_id(item.name, manager) + if o: + yield o + + diff --git a/thirdparty/boto/sdb/persist/checker.py b/thirdparty/boto/sdb/persist/checker.py new file mode 100644 index 000000000..47f736292 --- /dev/null +++ b/thirdparty/boto/sdb/persist/checker.py @@ -0,0 +1,303 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from datetime import datetime +import boto +from boto.s3.key import Key +from boto.s3.bucket import Bucket +from boto.sdb.persist import revive_object_from_id +from boto.exception import SDBPersistenceError +from boto.utils import Password + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class ValueChecker: + + def check(self, value): + """ + Checks a value to see if it is of the right type. + + Should raise a TypeError exception if an in appropriate value is passed in. + """ + raise TypeError + + def from_string(self, str_value, obj): + """ + Takes a string as input and returns the type-specific value represented by that string. + + Should raise a ValueError if the value cannot be converted to the appropriate type. + """ + raise ValueError + + def to_string(self, value): + """ + Convert a value to it's string representation. + + Should raise a ValueError if the value cannot be converted to a string representation. + """ + raise ValueError + +class StringChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.check(params['default']) + self.default = params['default'] + else: + self.default = '' + + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + + def from_string(self, str_value, obj): + return str_value + + def to_string(self, value): + self.check(value) + return value + +class PasswordChecker(StringChecker): + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode) or isinstance(value, Password): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + +class IntegerChecker(ValueChecker): + + __sizes__ = { 'small' : (65535, 32767, -32768, 5), + 'medium' : (4294967295, 2147483647, -2147483648, 10), + 'Large' : (18446744073709551615, 9223372036854775807, -9223372036854775808, 20)} + + def __init__(self, **params): + self.size = params.get('size', 'medium') + if self.size not in self.__sizes__.keys(): + raise ValueError, 'size must be one of %s' % self.__siz8es__.keys() + self.signed = params.get('signed', True) + self.default = params.get('default', 0) + self.format_string = '%%0%dd' % self.__sizes__[self.size][-1] + + def check(self, value): + if not isinstance(value, int) and not isinstance(value, long): + raise TypeError, 'Expecting int or long, got %s' % type(value) + if self.signed: + min = self.__sizes__[self.size][2] + max = self.__sizes__[self.size][1] + else: + min = 0 + max = self.__sizes__[self.size][0] + if value > max: + raise ValueError, 'Maximum value is %d' % max + if value < min: + raise ValueError, 'Minimum value is %d' % min + + def from_string(self, str_value, obj): + val = int(str_value) + if self.signed: + val = val + self.__sizes__[self.size][2] + return val + + def to_string(self, value): + self.check(value) + if self.signed: + value += -self.__sizes__[self.size][2] + return self.format_string % value + +class BooleanChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('default'): + self.default = params['default'] + else: + self.default = False + + def check(self, value): + if not isinstance(value, bool): + raise TypeError, 'Expecting bool, got %s' % type(value) + + def from_string(self, str_value, obj): + if str_value.lower() == 'true': + return True + else: + return False + + def to_string(self, value): + self.check(value) + if value == True: + return 'true' + else: + return 'false' + +class DateTimeChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.default = params['default'] + else: + self.default = datetime.now() + + def check(self, value): + if not isinstance(value, datetime): + raise TypeError, 'Expecting datetime, got %s' % type(value) + + def from_string(self, str_value, obj): + try: + return datetime.strptime(str_value, ISO8601) + except: + raise ValueError, 'Unable to convert %s to DateTime' % str_value + + def to_string(self, value): + self.check(value) + return value.strftime(ISO8601) + +class ObjectChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + self.ref_class = params.get('ref_class', None) + if self.ref_class == None: + raise SDBPersistenceError('ref_class parameter is required') + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + # ugly little hack - sometimes I want to just stick a UUID string + # in here rather than instantiate an object. + # This does a bit of hand waving to "type check" the string + t = value.split('-') + if len(t) != 5: + raise ValueError + else: + try: + obj_lineage = value.get_lineage() + cls_lineage = self.ref_class.get_lineage() + if obj_lineage.startswith(cls_lineage): + return + raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) + except: + raise ValueError, '%s is not an SDBObject' % value + + def from_string(self, str_value, obj): + if not str_value: + return None + try: + return revive_object_from_id(str_value, obj._manager) + except: + raise ValueError, 'Unable to convert %s to Object' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + +class S3KeyChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + try: + bucket_name, key_name = value.split('/', 1) + except: + raise ValueError + elif not isinstance(value, Key): + raise TypeError, 'Expecting Key, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + bucket_name, key_name = str_value.split('/', 1) + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(bucket_name) + key = bucket.get_key(key_name) + if not key: + key = bucket.new_key(key_name) + return key + except: + raise ValueError, 'Unable to convert %s to S3Key' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return '%s/%s' % (value.bucket.name, value.name) + +class S3BucketChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + return + elif not isinstance(value, Bucket): + raise TypeError, 'Expecting Bucket, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(str_value) + return bucket + except: + raise ValueError, 'Unable to convert %s to S3Bucket' % str_value + + def to_string(self, value): + self.check(value) + if value == None: + return '' + else: + return '%s' % value.name + diff --git a/thirdparty/boto/sdb/persist/object.py b/thirdparty/boto/sdb/persist/object.py new file mode 100644 index 000000000..114c03421 --- /dev/null +++ b/thirdparty/boto/sdb/persist/object.py @@ -0,0 +1,196 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, object_lister +from boto.sdb.persist.property import * +import uuid + +class SDBBase(type): + "Metaclass for all SDBObjects" + def __init__(cls, name, bases, dict): + super(SDBBase, cls).__init__(name, bases, dict) + # Make sure this is a subclass of SDBObject - mainly copied from django ModelBase (thanks!) + try: + if filter(lambda b: issubclass(b, SDBObject), bases): + # look for all of the Properties and set their names + for key in dict.keys(): + if isinstance(dict[key], Property): + property = dict[key] + property.set_name(key) + prop_names = [] + props = cls.properties() + for prop in props: + prop_names.append(prop.name) + setattr(cls, '_prop_names', prop_names) + except NameError: + # 'SDBObject' isn't defined yet, meaning we're looking at our own + # SDBObject class, defined below. + pass + +class SDBObject(object): + __metaclass__ = SDBBase + + _manager = get_manager() + + @classmethod + def get_lineage(cls): + l = [c.__name__ for c in cls.mro()] + l.reverse() + return '.'.join(l) + + @classmethod + def get(cls, id=None, **params): + if cls._manager.domain and id: + a = cls._manager.domain.get_attributes(id, '__type__') + if a.has_key('__type__'): + return cls(id) + else: + raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) + else: + rs = cls.find(**params) + try: + obj = rs.next() + except StopIteration: + raise SDBPersistenceError('%s object matching query does not exist' % cls.__name__) + try: + rs.next() + except StopIteration: + return obj + raise SDBPersistenceError('Query matched more than 1 item') + + @classmethod + def find(cls, **params): + keys = params.keys() + if len(keys) > 4: + raise SDBPersistenceError('Too many fields, max is 4') + parts = ["['__type__'='%s'] union ['__lineage__'starts-with'%s']" % (cls.__name__, cls.get_lineage())] + properties = cls.properties() + for key in keys: + found = False + for property in properties: + if property.name == key: + found = True + if isinstance(property, ScalarProperty): + checker = property.checker + parts.append("['%s' = '%s']" % (key, checker.to_string(params[key]))) + else: + raise SDBPersistenceError('%s is not a searchable field' % key) + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + query = ' intersection '.join(parts) + if cls._manager.domain: + rs = cls._manager.domain.query(query) + else: + rs = [] + return object_lister(None, rs, cls._manager) + + @classmethod + def list(cls, max_items=None): + if cls._manager.domain: + rs = cls._manager.domain.query("['__type__' = '%s']" % cls.__name__, max_items=max_items) + else: + rs = [] + return object_lister(cls, rs, cls._manager) + + @classmethod + def properties(cls): + properties = [] + while cls: + for key in cls.__dict__.keys(): + if isinstance(cls.__dict__[key], ScalarProperty): + properties.append(cls.__dict__[key]) + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return properties + + # for backwards compatibility + find_properties = properties + + def __init__(self, id=None, manager=None): + if manager: + self._manager = manager + self.id = id + if self.id: + self._auto_update = True + if self._manager.domain: + attrs = self._manager.domain.get_attributes(self.id, '__type__') + if len(attrs.keys()) == 0: + raise SDBPersistenceError('Object %s: not found' % self.id) + else: + self.id = str(uuid.uuid4()) + self._auto_update = False + + def __setattr__(self, name, value): + if name in self._prop_names: + object.__setattr__(self, name, value) + elif name.startswith('_'): + object.__setattr__(self, name, value) + elif name == 'id': + object.__setattr__(self, name, value) + else: + self._persist_attribute(name, value) + object.__setattr__(self, name, value) + + def __getattr__(self, name): + if not name.startswith('_'): + a = self._manager.domain.get_attributes(self.id, name) + if a.has_key(name): + object.__setattr__(self, name, a[name]) + return a[name] + raise AttributeError + + def __repr__(self): + return '%s<%s>' % (self.__class__.__name__, self.id) + + def _persist_attribute(self, name, value): + if self.id: + self._manager.domain.put_attributes(self.id, {name : value}, replace=True) + + def _get_sdb_item(self): + return self._manager.domain.get_item(self.id) + + def save(self): + attrs = {'__type__' : self.__class__.__name__, + '__module__' : self.__class__.__module__, + '__lineage__' : self.get_lineage()} + for property in self.properties(): + attrs[property.name] = property.to_string(self) + if self._manager.domain: + self._manager.domain.put_attributes(self.id, attrs, replace=True) + self._auto_update = True + + def delete(self): + if self._manager.domain: + self._manager.domain.delete_attributes(self.id) + + def get_related_objects(self, ref_name, ref_cls=None): + if self._manager.domain: + query = "['%s' = '%s']" % (ref_name, self.id) + if ref_cls: + query += " intersection ['__type__'='%s']" % ref_cls.__name__ + rs = self._manager.domain.query(query) + else: + rs = [] + return object_lister(ref_cls, rs, self._manager) + diff --git a/thirdparty/boto/sdb/persist/property.py b/thirdparty/boto/sdb/persist/property.py new file mode 100644 index 000000000..d36b16010 --- /dev/null +++ b/thirdparty/boto/sdb/persist/property.py @@ -0,0 +1,354 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist.checker import * +from boto.utils import Password + +class Property(object): + + def __init__(self, checker_class, **params): + self.name = '' + self.checker = checker_class(**params) + self.slot_name = '__' + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + +class ScalarProperty(Property): + + def save(self, obj): + domain = obj._manager.domain + domain.put_attributes(obj.id, {self.name : self.to_string(obj)}, replace=True) + + def to_string(self, obj): + return self.checker.to_string(getattr(obj, self.name)) + + def load(self, obj): + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + # try to get the attribute value from SDB + if self.name in a: + value = self.checker.from_string(a[self.name], obj) + setattr(obj, self.slot_name, value) + # if it's not there, set the value to the default value + else: + self.__set__(obj, self.checker.default) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = self.checker.default + setattr(obj, self.slot_name, self.checker.default) + return value + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class StringProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, StringChecker, **params) + +class PasswordProperty(ScalarProperty): + """ + Hashed password + """ + + def __init__(self, **params): + ScalarProperty.__init__(self, PasswordChecker, **params) + + def __set__(self, obj, value): + p = Password() + p.set(value) + ScalarProperty.__set__(self, obj, p) + + def __get__(self, obj, objtype): + return Password(ScalarProperty.__get__(self, obj, objtype)) + +class SmallPositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class SmallIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class IntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class BooleanProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, BooleanChecker, **params) + +class DateTimeProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, DateTimeChecker, **params) + +class ObjectProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, ObjectChecker, **params) + +class S3KeyProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3KeyChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class S3BucketProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3BucketChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class MultiValueProperty(Property): + + def __init__(self, checker_class, **params): + Property.__init__(self, checker_class, **params) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = MultiValue(self, obj, []) + setattr(obj, self.slot_name, value) + return value + + def load(self, obj): + if obj != None: + _list = [] + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + if self.name in a: + lst = a[self.name] + if not isinstance(lst, list): + lst = [lst] + for value in lst: + value = self.checker.from_string(value, obj) + _list.append(value) + setattr(obj, self.slot_name, MultiValue(self, obj, _list)) + + def __set__(self, obj, value): + if not isinstance(value, list): + raise SDBPersistenceError('Value must be a list') + self._list = value + str_list = [] + for value in self._list: + str_list.append(self.checker.to_string(value)) + domain = obj._manager.domain + try: + domain.put_attributes(obj.id, {self.name : str_list}, replace=True) + except: + print 'problem setting value: %s' % value + +class StringListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, StringChecker, **params) + +class SmallIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class SmallPositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class IntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class BooleanListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, BooleanChecker, **params) + +class ObjectListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, ObjectChecker, **params) + +class HasManyProperty(Property): + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + + def __get__(self, obj, objtype): + return self + + +class MultiValue: + """ + Special Multi Value for boto persistence layer to allow us to do + obj.list.append(foo) + """ + def __init__(self, property, obj, _list): + self.checker = property.checker + self.name = property.name + self.object = obj + self._list = _list + + def __repr__(self): + return repr(self._list) + + def __getitem__(self, key): + return self._list.__getitem__(key) + + def __delitem__(self, key): + item = self[key] + self._list.__delitem__(key) + domain = self.object._manager.domain + domain.delete_attributes(self.object.id, {self.name: [self.checker.to_string(item)]}) + + def __len__(self): + return len(self._list) + + def append(self, value): + self.checker.check(value) + self._list.append(value) + domain = self.object._manager.domain + domain.put_attributes(self.object.id, {self.name: self.checker.to_string(value)}, replace=False) + + def index(self, value): + for x in self._list: + if x.id == value.id: + return self._list.index(x) + + def remove(self, value): + del(self[self.index(value)]) diff --git a/thirdparty/boto/sdb/persist/test_persist.py b/thirdparty/boto/sdb/persist/test_persist.py new file mode 100644 index 000000000..3207e58a7 --- /dev/null +++ b/thirdparty/boto/sdb/persist/test_persist.py @@ -0,0 +1,138 @@ +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import * +from boto.sdb.persist import Manager +from datetime import datetime +import time + +# +# This will eventually be moved to the boto.tests module and become a real unit test +# but for now it will live here. It shows examples of each of the Property types in +# use and tests the basic operations. +# +class TestScalar(SDBObject): + + name = StringProperty() + description = StringProperty() + size = PositiveIntegerProperty() + offset = IntegerProperty() + foo = BooleanProperty() + date = DateTimeProperty() + file = S3KeyProperty() + +class TestRef(SDBObject): + + name = StringProperty() + ref = ObjectProperty(ref_class=TestScalar) + +class TestSubClass1(TestRef): + + answer = PositiveIntegerProperty() + +class TestSubClass2(TestScalar): + + flag = BooleanProperty() + +class TestList(SDBObject): + + names = StringListProperty() + numbers = PositiveIntegerListProperty() + bools = BooleanListProperty() + objects = ObjectListProperty(ref_class=TestScalar) + +def test1(): + s = TestScalar() + s.name = 'foo' + s.description = 'This is foo' + s.size = 42 + s.offset = -100 + s.foo = True + s.date = datetime.now() + s.save() + return s + +def test2(ref_name): + s = TestRef() + s.name = 'testref' + rs = TestScalar.find(name=ref_name) + s.ref = rs.next() + s.save() + return s + +def test3(): + s = TestScalar() + s.name = 'bar' + s.description = 'This is bar' + s.size = 24 + s.foo = False + s.date = datetime.now() + s.save() + return s + +def test4(ref1, ref2): + s = TestList() + s.names.append(ref1.name) + s.names.append(ref2.name) + s.numbers.append(ref1.size) + s.numbers.append(ref2.size) + s.bools.append(ref1.foo) + s.bools.append(ref2.foo) + s.objects.append(ref1) + s.objects.append(ref2) + s.save() + return s + +def test5(ref): + s = TestSubClass1() + s.answer = 42 + s.ref = ref + s.save() + # test out free form attribute + s.fiddlefaddle = 'this is fiddlefaddle' + s._fiddlefaddle = 'this is not fiddlefaddle' + return s + +def test6(): + s = TestSubClass2() + s.name = 'fie' + s.description = 'This is fie' + s.size = 4200 + s.offset = -820 + s.foo = False + s.date = datetime.now() + s.flag = True + s.save() + return s + +def test(domain_name): + print 'Initialize the Persistance system' + Manager.DefaultDomainName = domain_name + print 'Call test1' + s1 = test1() + # now create a new instance and read the saved data from SDB + print 'Now sleep to wait for things to converge' + time.sleep(5) + print 'Now lookup the object and compare the fields' + s2 = TestScalar(s1.id) + assert s1.name == s2.name + assert s1.description == s2.description + assert s1.size == s2.size + assert s1.offset == s2.offset + assert s1.foo == s2.foo + #assert s1.date == s2.date + print 'Call test2' + s2 = test2(s1.name) + print 'Call test3' + s3 = test3() + print 'Call test4' + s4 = test4(s1, s3) + print 'Call test5' + s6 = test6() + s5 = test5(s6) + domain = s5._manager.domain + item1 = domain.get_item(s1.id) + item2 = domain.get_item(s2.id) + item3 = domain.get_item(s3.id) + item4 = domain.get_item(s4.id) + item5 = domain.get_item(s5.id) + item6 = domain.get_item(s6.id) + return [(s1, item1), (s2, item2), (s3, item3), (s4, item4), (s5, item5), (s6, item6)] diff --git a/thirdparty/boto/sdb/queryresultset.py b/thirdparty/boto/sdb/queryresultset.py new file mode 100644 index 000000000..776a97a0d --- /dev/null +++ b/thirdparty/boto/sdb/queryresultset.py @@ -0,0 +1,50 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sdb.item import Item + +def query_lister(domain, query='', max_items=None): + more_results = True + num_results = 0 + next_token = None + while more_results: + rs = domain.connection.query(domain.name, query, None, next_token) + for item_name in rs: + if max_items: + if num_results == max_items: + raise StopIteration + yield Item(domain, item_name) + num_results += 1 + next_token = rs.next_token + more_results = next_token != None + +class QueryResultSet: + + def __init__(self, domain=None, query='', max_items=None): + self.max_items = max_items + self.domain = domain + self.query = query + + def __iter__(self): + return query_lister(self.domain, self.query, self.max_items) + + + diff --git a/thirdparty/boto/services/__init__.py b/thirdparty/boto/services/__init__.py new file mode 100644 index 000000000..449bd162a --- /dev/null +++ b/thirdparty/boto/services/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/services/bs.py b/thirdparty/boto/services/bs.py new file mode 100644 index 000000000..3176c519b --- /dev/null +++ b/thirdparty/boto/services/bs.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from optparse import OptionParser +from boto.services.servicedef import ServiceDef +from boto.services.message import ServiceMessage +from boto.services.submit import Submitter +from boto.services.result import ResultProcessor +import boto +import sys, os, StringIO + +class BS(object): + + Usage = "usage: %prog [options] config_file command" + + Commands = {'reset' : 'Clear input queue and output bucket', + 'submit' : 'Submit local files to the service', + 'start' : 'Start the service', + 'status' : 'Report on the status of the service buckets and queues', + 'retrieve' : 'Retrieve output generated by a batch', + 'batches' : 'List all batches stored in current output_domain'} + + def __init__(self): + self.service_name = None + self.parser = OptionParser(usage=self.Usage) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option("-a", "--access-key", action="store", type="string", + help="your AWS Access Key") + self.parser.add_option("-s", "--secret-key", action="store", type="string", + help="your AWS Secret Access Key") + self.parser.add_option("-p", "--path", action="store", type="string", dest="path", + help="the path to local directory for submit and retrieve") + self.parser.add_option("-k", "--keypair", action="store", type="string", dest="keypair", + help="the SSH keypair used with launched instance(s)") + self.parser.add_option("-l", "--leave", action="store_true", dest="leave", + help="leave the files (don't retrieve) files during retrieve command") + self.parser.set_defaults(leave=False) + self.parser.add_option("-n", "--num-instances", action="store", type="string", dest="num_instances", + help="the number of launched instance(s)") + self.parser.set_defaults(num_instances=1) + self.parser.add_option("-i", "--ignore-dirs", action="append", type="string", dest="ignore", + help="directories that should be ignored by submit command") + self.parser.add_option("-b", "--batch-id", action="store", type="string", dest="batch", + help="batch identifier required by the retrieve command") + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_reset(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'clearing out input queue' + i = 0 + m = iq.read() + while m: + i += 1 + iq.delete_message(m) + m = iq.read() + print 'deleted %d messages' % i + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + print 'delete generated files in output bucket' + i = 0 + for k in ob: + i += 1 + k.delete() + print 'deleted %d keys' % i + + def do_submit(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + s = Submitter(self.sd) + t = s.submit_path(self.options.path, None, self.options.ignore, None, + None, True, self.options.path) + print 'A total of %d files were submitted' % t[1] + print 'Batch Identifier: %s' % t[0] + + def do_start(self): + ami_id = self.sd.get('ami_id') + if not ami_id: + self.parser.error('ami_id option is required when starting the service') + ec2 = boto.connect_ec2() + if not self.sd.has_section('Credentials'): + self.sd.add_section('Credentials') + self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id) + self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key) + s = StringIO.StringIO() + self.sd.write(s) + rs = ec2.get_all_images([ami_id]) + img = rs[0] + r = img.run(user_data=s.getvalue(), key_name=self.options.keypair, + max_count=self.options.num_instances) + print 'Starting AMI: %s' % ami_id + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + + def do_status(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'The input_queue (%s) contains approximately %s messages' % (iq.id, iq.count()) + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + total = 0 + for k in ob: + total += 1 + print 'The output_bucket (%s) contains %d keys' % (ob.name, total) + + def do_retrieve(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + if not self.options.batch: + self.parser.error('batch identifier is required for retrieve command') + s = ResultProcessor(self.options.batch, self.sd) + s.get_results(self.options.path, get_file=(not self.options.leave)) + + def do_batches(self): + d = self.sd.get_obj('output_domain') + if d: + print 'Available Batches:' + rs = d.query("['type'='Batch']") + for item in rs: + print ' %s' % item.name + else: + self.parser.error('No output_domain specified for service') + + def main(self): + self.options, self.args = self.parser.parse_args() + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) != 2: + self.parser.error("config_file and command are required") + self.config_file = self.args[0] + self.sd = ServiceDef(self.config_file) + self.command = self.args[1] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + +if __name__ == "__main__": + bs = BS() + bs.main() diff --git a/thirdparty/boto/services/message.py b/thirdparty/boto/services/message.py new file mode 100644 index 000000000..6bb2e58ab --- /dev/null +++ b/thirdparty/boto/services/message.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.sqs.message import MHMessage +from boto.utils import get_ts +from socket import gethostname +import os, mimetypes, time + +class ServiceMessage(MHMessage): + + def for_key(self, key, params=None, bucket_name=None): + if params: + self.update(params) + if key.path: + t = os.path.split(key.path) + self['OriginalLocation'] = t[0] + self['OriginalFileName'] = t[1] + mime_type = mimetypes.guess_type(t[1])[0] + if mime_type == None: + mime_type = 'application/octet-stream' + self['Content-Type'] = mime_type + s = os.stat(key.path) + t = time.gmtime(s[7]) + self['FileAccessedDate'] = get_ts(t) + t = time.gmtime(s[8]) + self['FileModifiedDate'] = get_ts(t) + t = time.gmtime(s[9]) + self['FileCreateDate'] = get_ts(t) + else: + self['OriginalFileName'] = key.name + self['OriginalLocation'] = key.bucket.name + self['ContentType'] = key.content_type + self['Host'] = gethostname() + if bucket_name: + self['Bucket'] = bucket_name + else: + self['Bucket'] = key.bucket.name + self['InputKey'] = key.name + self['Size'] = key.size + diff --git a/thirdparty/boto/services/result.py b/thirdparty/boto/services/result.py new file mode 100644 index 000000000..5ef9434c8 --- /dev/null +++ b/thirdparty/boto/services/result.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import getopt, sys, os, time, mimetypes +from datetime import datetime, timedelta +from boto.services.servicedef import ServiceDef +from boto.utils import parse_ts +import boto + +class ResultProcessor: + + LogFileName = 'log.csv' + + def __init__(self, batch_name, sd, mimetype_files=None): + self.sd = sd + self.batch = batch_name + self.log_fp = None + self.num_files = 0 + self.total_time = 0 + self.min_time = timedelta.max + self.max_time = timedelta.min + self.earliest_time = datetime.max + self.latest_time = datetime.min + self.queue = self.sd.get_obj('output_queue') + self.domain = self.sd.get_obj('output_domain') + + def calculate_stats(self, msg): + start_time = parse_ts(msg['Service-Read']) + end_time = parse_ts(msg['Service-Write']) + elapsed_time = end_time - start_time + if elapsed_time > self.max_time: + self.max_time = elapsed_time + if elapsed_time < self.min_time: + self.min_time = elapsed_time + self.total_time += elapsed_time.seconds + if start_time < self.earliest_time: + self.earliest_time = start_time + if end_time > self.latest_time: + self.latest_time = end_time + + def log_message(self, msg, path): + keys = msg.keys() + keys.sort() + if not self.log_fp: + self.log_fp = open(os.path.join(path, self.LogFileName), 'w') + line = ','.join(keys) + self.log_fp.write(line+'\n') + values = [] + for key in keys: + value = msg[key] + if value.find(',') > 0: + value = '"%s"' % value + values.append(value) + line = ','.join(values) + self.log_fp.write(line+'\n') + + def process_record(self, record, path, get_file=True): + self.log_message(record, path) + self.calculate_stats(record) + outputs = record['OutputKey'].split(',') + if record.has_key('OutputBucket'): + bucket = boto.lookup('s3', record['OutputBucket']) + else: + bucket = boto.lookup('s3', record['Bucket']) + for output in outputs: + if get_file: + key_name, type = output.split(';') + if type: + mimetype = type.split('=')[1] + key = bucket.lookup(key_name) + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) + key.get_contents_to_filename(file_name) + self.num_files += 1 + + def get_results_from_queue(self, path, get_file=True, delete_msg=True): + m = self.queue.read() + while m: + if m['Batch'] == self.batch: + self.process_record(m, path, get_file) + if delete_msg: + self.queue.delete_message(m) + m = self.queue.read() + + def get_results_from_domain(self, path, get_file=True): + rs = self.domain.query("['Batch'='%s']" % self.batch) + for item in rs: + self.process_record(item, path, get_file) + + def get_results_from_bucket(self, path): + bucket = self.sd.get_obj('output_bucket') + if bucket: + print 'No output queue or domain, just retrieving files from output_bucket' + for key in bucket: + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) + key.get_contents_to_filename(file_name) + self.num_files + 1 + + def get_results(self, path, get_file=True, delete_msg=True): + if not os.path.isdir(path): + os.mkdir(path) + if self.queue: + self.get_results_from_queue(path, get_file) + elif self.domain: + self.get_results_from_domain(path, get_file) + else: + self.get_results_from_bucket(path) + if self.log_fp: + self.log_fp.close() + print '%d results successfully retrieved.' % self.num_files + if self.num_files > 0: + self.avg_time = float(self.total_time)/self.num_files + print 'Minimum Processing Time: %d' % self.min_time.seconds + print 'Maximum Processing Time: %d' % self.max_time.seconds + print 'Average Processing Time: %f' % self.avg_time + self.elapsed_time = self.latest_time-self.earliest_time + print 'Elapsed Time: %d' % self.elapsed_time.seconds + tput = 1.0 / ((self.elapsed_time.seconds/60.0) / self.num_files) + print 'Throughput: %f transactions / minute' % tput + diff --git a/thirdparty/boto/services/service.py b/thirdparty/boto/services/service.py new file mode 100644 index 000000000..c35fa26d5 --- /dev/null +++ b/thirdparty/boto/services/service.py @@ -0,0 +1,165 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.message import ServiceMessage +from boto.services.servicedef import ServiceDef +from boto.pyami.scriptbase import ScriptBase +from boto.exception import S3ResponseError +from boto.utils import get_ts +import StringIO +import time +import os +import sys, traceback +import md5 +import mimetypes + +class Service(ScriptBase): + + # Time required to process a transaction + ProcessingTime = 60 + + def __init__(self, config_file=None, mimetype_files=None): + ScriptBase.__init__(self, config_file) + self.name = self.__class__.__name__ + self.working_dir = boto.config.get('Pyami', 'working_dir') + self.sd = ServiceDef(config_file) + self.retry_count = self.sd.getint('retry_count', 5) + self.loop_delay = self.sd.getint('loop_delay', 30) + self.processing_time = self.sd.getint('processing_time', 60) + self.input_queue = self.sd.get_obj('input_queue') + self.output_queue = self.sd.get_obj('output_queue') + self.output_domain = self.sd.get_obj('output_domain') + if mimetype_files: + mimetypes.init(mimetype_files) + + def split_key(key): + if key.find(';') < 0: + t = (key, '') + else: + key, type = key.split(';') + label, mtype = type.split('=') + t = (key, mtype) + return t + + def read_message(self): + boto.log.info('read_message') + message = self.input_queue.read(self.processing_time) + if message: + boto.log.info(message.get_body()) + key = 'Service-Read' + message[key] = get_ts() + return message + + # retrieve the source file from S3 + def get_file(self, message): + bucket_name = message['Bucket'] + key_name = message['InputKey'] + file_name = os.path.join(self.working_dir, message.get('OriginalFileName', 'in_file')) + boto.log.info('get_file: %s/%s to %s' % (bucket_name, key_name, file_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.get_contents_to_filename(os.path.join(self.working_dir, file_name)) + return file_name + + # process source file, return list of output files + def process_file(self, in_file_name, msg): + return [] + + # store result file in S3 + def put_file(self, bucket_name, file_path, key_name=None): + boto.log.info('putting file %s as %s.%s' % (file_path, bucket_name, key_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.set_contents_from_filename(file_path) + return key + + def save_results(self, results, input_message, output_message): + output_keys = [] + for file, type in results: + if input_message.has_key('OutputBucket'): + output_bucket = input_message['OutputBucket'] + else: + output_bucket = input_message['Bucket'] + key_name = os.path.split(file)[1] + key = self.put_file(output_bucket, file, key_name) + output_keys.append('%s;type=%s' % (key.name, type)) + output_message['OutputKey'] = ','.join(output_keys) + + # write message to each output queue + def write_message(self, message): + message['Service-Write'] = get_ts() + message['Server'] = self.name + if os.environ.has_key('HOSTNAME'): + message['Host'] = os.environ['HOSTNAME'] + else: + message['Host'] = 'unknown' + message['Instance-ID'] = self.instance_id + if self.output_queue: + boto.log.info('Writing message to SQS queue: %s' % self.output_queue.id) + self.output_queue.write(message) + if self.output_domain: + boto.log.info('Writing message to SDB domain: %s' % self.output_domain.name) + item_name = '/'.join([message['Service-Write'], message['Bucket'], message['InputKey']]) + self.output_domain.put_attributes(item_name, message) + + # delete message from input queue + def delete_message(self, message): + boto.log.info('deleting message from %s' % self.input_queue.id) + self.input_queue.delete_message(message) + + # to clean up any files, etc. after each iteration + def cleanup(self): + pass + + def shutdown(self): + on_completion = self.sd.get('on_completion', 'shutdown') + if on_completion == 'shutdown': + if self.instance_id: + time.sleep(60) + c = boto.connect_ec2() + c.terminate_instances([self.instance_id]) + + def main(self, notify=False): + self.notify('Service: %s Starting' % self.name) + empty_reads = 0 + while self.retry_count < 0 or empty_reads < self.retry_count: + try: + input_message = self.read_message() + if input_message: + empty_reads = 0 + output_message = ServiceMessage(None, input_message.get_body()) + input_file = self.get_file(input_message) + results = self.process_file(input_file, output_message) + self.save_results(results, input_message, output_message) + self.write_message(output_message) + self.delete_message(input_message) + self.cleanup() + else: + empty_reads += 1 + time.sleep(self.loop_delay) + except Exception, e: + boto.log.exception('Service Failed') + empty_reads += 1 + self.create_connections() + self.notify('Service: %s Shutting Down' % self.name) + self.shutdown() + diff --git a/thirdparty/boto/services/servicedef.py b/thirdparty/boto/services/servicedef.py new file mode 100644 index 000000000..1cb01aa75 --- /dev/null +++ b/thirdparty/boto/services/servicedef.py @@ -0,0 +1,91 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.pyami.config import Config +from boto.services.message import ServiceMessage +import boto + +class ServiceDef(Config): + + def __init__(self, config_file, aws_access_key_id=None, aws_secret_access_key=None): + Config.__init__(self, config_file) + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + script = Config.get(self, 'Pyami', 'scripts') + if script: + self.name = script.split('.')[-1] + else: + self.name = None + + + def get(self, name, default=None): + return Config.get(self, self.name, name, default) + + def has_option(self, option): + return Config.has_option(self, self.name, option) + + def getint(self, option, default=0): + try: + val = Config.get(self, self.name, option) + val = int(val) + except: + val = int(default) + return val + + def getbool(self, option, default=False): + try: + val = Config.get(self, self.name, option) + if val.lower() == 'true': + val = True + else: + val = False + except: + val = default + return val + + def get_obj(self, name): + """ + Returns the AWS object associated with a given option. + + The heuristics used are a bit lame. If the option name contains + the word 'bucket' it is assumed to be an S3 bucket, if the name + contains the word 'queue' it is assumed to be an SQS queue and + if it contains the word 'domain' it is assumed to be a SimpleDB + domain. If the option name specified does not exist in the + config file or if the AWS object cannot be retrieved this + returns None. + """ + val = self.get(name) + if not val: + return None + if name.find('queue') >= 0: + obj = boto.lookup('sqs', val) + if obj: + obj.set_message_class(ServiceMessage) + elif name.find('bucket') >= 0: + obj = boto.lookup('s3', val) + elif name.find('domain') >= 0: + obj = boto.lookup('sdb', val) + else: + obj = None + return obj + + diff --git a/thirdparty/boto/services/sonofmmm.cfg b/thirdparty/boto/services/sonofmmm.cfg new file mode 100644 index 000000000..d70d3794d --- /dev/null +++ b/thirdparty/boto/services/sonofmmm.cfg @@ -0,0 +1,43 @@ +# +# Your AWS Credentials +# You only need to supply these in this file if you are not using +# the boto tools to start your service +# +#[Credentials] +#aws_access_key_id = +#aws_secret_access_key = + +# +# Fill out this section if you want emails from the service +# when it starts and stops +# +#[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +[Pyami] +scripts = boto.services.sonofmmm.SonOfMMM + +[SonOfMMM] +# id of the AMI to be launched +ami_id = ami-dc799cb5 +# number of times service will read an empty queue before exiting +# a negative value will cause the service to run forever +retry_count = 5 +# seconds to wait after empty queue read before reading again +loop_delay = 10 +# average time it takes to process a transaction +# controls invisibility timeout of messages +processing_time = 60 +ffmpeg_args = -y -i %%s -f mov -r 29.97 -b 1200kb -mbd 2 -flags +4mv+trell -aic 2 -cmp 2 -subcmp 2 -ar 48000 -ab 19200 -s 320x240 -vcodec mpeg4 -acodec libfaac %%s +output_mimetype = video/quicktime +output_ext = .mov +input_bucket = +output_bucket = +output_domain = +output_queue = +input_queue = + diff --git a/thirdparty/boto/services/sonofmmm.py b/thirdparty/boto/services/sonofmmm.py new file mode 100644 index 000000000..5b94f902c --- /dev/null +++ b/thirdparty/boto/services/sonofmmm.py @@ -0,0 +1,80 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.service import Service +from boto.services.message import ServiceMessage +import os, time, mimetypes + +class SonOfMMM(Service): + + def __init__(self, config_file=None): + Service.__init__(self, config_file) + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.working_dir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + if self.sd.has_option('ffmpeg_args'): + self.command = '/usr/local/bin/ffmpeg ' + self.sd.get('ffmpeg_args') + else: + self.command = '/usr/local/bin/ffmpeg -y -i %s %s' + self.output_mimetype = self.sd.get('output_mimetype') + if self.sd.has_option('output_ext'): + self.output_ext = self.sd.get('output_ext') + else: + self.output_ext = mimetypes.guess_extension(self.output_mimetype) + self.output_bucket = self.sd.get_obj('output_bucket') + self.input_bucket = self.sd.get_obj('input_bucket') + # check to see if there are any messages queue + # if not, create messages for all files in input_bucket + m = self.input_queue.read(1) + if not m: + self.queue_files() + + def queue_files(self): + boto.log.info('Queueing files from %s' % self.input_bucket.name) + for key in self.input_bucket: + boto.log.info('Queueing %s' % key.name) + m = ServiceMessage() + if self.output_bucket: + d = {'OutputBucket' : self.output_bucket.name} + else: + d = None + m.for_key(key, d) + self.input_queue.write(m) + + def process_file(self, in_file_name, msg): + base, ext = os.path.splitext(in_file_name) + out_file_name = os.path.join(self.working_dir, + base+self.output_ext) + command = self.command % (in_file_name, out_file_name) + boto.log.info('running:\n%s' % command) + status = self.run(command) + if status == 0: + return [(out_file_name, self.output_mimetype)] + else: + return [] + + def shutdown(self): + if os.path.isfile(self.log_path): + if self.output_bucket: + key = self.output_bucket.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + Service.shutdown(self) diff --git a/thirdparty/boto/services/submit.py b/thirdparty/boto/services/submit.py new file mode 100644 index 000000000..dfa71f2a0 --- /dev/null +++ b/thirdparty/boto/services/submit.py @@ -0,0 +1,87 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +import time, os + +class Submitter: + + def __init__(self, sd): + self.sd = sd + self.input_bucket = self.sd.get_obj('input_bucket') + self.output_bucket = self.sd.get_obj('output_bucket') + self.output_domain = self.sd.get_obj('output_domain') + self.queue = self.sd.get_obj('input_queue') + + def get_key_name(self, fullpath, prefix): + key_name = fullpath[len(prefix):] + l = key_name.split(os.sep) + return '/'.join(l) + + def write_message(self, key, metadata): + if self.queue: + m = self.queue.new_message() + m.for_key(key, metadata) + if self.output_bucket: + m['OutputBucket'] = self.output_bucket.name + self.queue.write(m) + + def submit_file(self, path, metadata=None, cb=None, num_cb=0, prefix='/'): + if not metadata: + metadata = {} + key_name = self.get_key_name(path, prefix) + k = self.input_bucket.new_key(key_name) + k.update_metadata(metadata) + k.set_contents_from_filename(path, replace=False, cb=cb, num_cb=num_cb) + self.write_message(k, metadata) + + def submit_path(self, path, tags=None, ignore_dirs=None, cb=None, num_cb=0, status=False, prefix='/'): + path = os.path.expanduser(path) + path = os.path.expandvars(path) + path = os.path.abspath(path) + total = 0 + metadata = {} + if tags: + metadata['Tags'] = tags + l = [] + for t in time.gmtime(): + l.append(str(t)) + metadata['Batch'] = '_'.join(l) + if self.output_domain: + self.output_domain.put_attributes(metadata['Batch'], {'type' : 'Batch'}) + if os.path.isdir(path): + for root, dirs, files in os.walk(path): + if ignore_dirs: + for ignore in ignore_dirs: + if ignore in dirs: + dirs.remove(ignore) + for file in files: + fullpath = os.path.join(root, file) + if status: + print 'Submitting %s' % fullpath + self.submit_file(fullpath, metadata, cb, num_cb, prefix) + total += 1 + elif os.path.isfile(path): + self.submit_file(path, metadata, cb, num_cb) + total += 1 + else: + print 'problem with %s' % path + return (metadata['Batch'], total) diff --git a/thirdparty/boto/sqs/20070501/__init__.py b/thirdparty/boto/sqs/20070501/__init__.py new file mode 100644 index 000000000..561f9cfff --- /dev/null +++ b/thirdparty/boto/sqs/20070501/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from connection import SQSConnection as Connection +from queue import Queue +from message import Message, MHMessage + +__all__ = ['Connection', 'Queue', 'Message', 'MHMessage'] diff --git a/thirdparty/boto/sqs/20070501/attributes.py b/thirdparty/boto/sqs/20070501/attributes.py new file mode 100644 index 000000000..b13370df0 --- /dev/null +++ b/thirdparty/boto/sqs/20070501/attributes.py @@ -0,0 +1,45 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Attribute Name/Value set +""" + +class Attributes(dict): + + def __init__(self): + self.current_key = None + self.current_value = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'AttributedValue': + self[self.current_key] = self.current_value + elif name == 'Attribute': + self.current_key = value + elif name == 'Value': + self.current_value = value + else: + setattr(self, name, value) + + diff --git a/thirdparty/boto/sqs/20070501/connection.py b/thirdparty/boto/sqs/20070501/connection.py new file mode 100644 index 000000000..7890aa2ed --- /dev/null +++ b/thirdparty/boto/sqs/20070501/connection.py @@ -0,0 +1,389 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSAuthConnection, AWSQueryConnection +import xml.sax +from boto.sqs.queue import Queue +from boto.sqs.message import Message +from boto.sqs.attributes import Attributes +from boto import handler +from boto.resultset import ResultSet +from boto.exception import SQSError + +PERM_ReceiveMessage = 'ReceiveMessage' +PERM_SendMessage = 'SendMessage' +PERM_FullControl = 'FullControl' + +AllPermissions = [PERM_ReceiveMessage, PERM_SendMessage, PERM_FullControl] + +class SQSQueryConnection(AWSQueryConnection): + + """ + This class uses the Query API (boo!) to SQS to access some of the + new features which have not yet been added to the REST api (yeah!). + """ + + DefaultHost = 'queue.amazonaws.com' + APIVersion = '2007-05-01' + SignatureVersion = '1' + DefaultContentType = 'text/plain' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0, https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + host, debug, https_connection_factory) + + def get_queue_attributes(self, queue_url, attribute='All'): + params = {'Attribute' : attribute} + response = self.make_request('GetQueueAttributes', params, queue_url) + body = response.read() + if response.status == 200: + attrs = Attributes() + h = handler.XmlHandler(attrs, self) + xml.sax.parseString(body, h) + return attrs + else: + raise SQSError(response.status, response.reason, body) + + def set_queue_attribute(self, queue_url, attribute, value): + params = {'Attribute' : attribute, 'Value' : value} + response = self.make_request('SetQueueAttributes', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def change_message_visibility(self, queue_url, message_id, vtimeout): + params = {'MessageId' : message_id, + 'VisibilityTimeout' : vtimeout} + response = self.make_request('ChangeMessageVisibility', params, + queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def add_grant(self, queue_url, permission, email_address=None, user_id=None): + params = {'Permission' : permission} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + response = self.make_request('AddGrant', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def remove_grant(self, queue_url, permission, email_address=None, user_id=None): + params = {'Permission' : permission} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + response = self.make_request('RemoveGrant', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + + def list_grants(self, queue_url, permission=None, email_address=None, user_id=None): + params = {} + if user_id: + params['Grantee.ID'] = user_id + if email_address: + params['Grantee.EmailAddress'] = email_address + if permission: + params['Permission'] = permission + response = self.make_request('ListGrants', params, queue_url) + body = response.read() + if response.status == 200: + return body + else: + raise SQSError(response.status, response.reason, body) + + def receive_message(self, queue_url, number_messages=1, + visibility_timeout=None, message_class=Message): + """ + This provides the same functionality as the read and get_messages methods + of the queue object. The only reason this is included here is that there is + currently a bug in SQS that makes it impossible to read a message from a queue + owned by someone else (even if you have been granted appropriate permissions) + via the REST interface. As it turns out, I need to be able to do this so until + the REST interface gets fixed this is the workaround. + """ + params = {'NumberOfMessages' : number_messages} + if visibility_timeout: + params['VisibilityTimeout'] = visibility_timeout + response = self.make_request('ReceiveMessage', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet([('Message', message_class)]) + h = handler.XmlHandler(rs, queue_url) + xml.sax.parseString(body, h) + if len(rs) == 1: + return rs[0] + else: + return rs + else: + raise SQSError(response.status, response.reason, body) + + def delete_message(self, queue_url, message_id): + """ + Because we have to use the Query interface to read messages from queues that + we don't own, we also have to provide a way to delete those messages via Query. + """ + params = {'MessageId' : message_id} + response = self.make_request('DeleteMessage', params, queue_url) + body = response.read() + if response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.status + else: + raise SQSError(response.status, response.reason, body) + +class SQSConnection(AWSAuthConnection): + + DefaultHost = 'queue.amazonaws.com' + APIVersion = '2007-05-01' + DefaultContentType = 'text/plain' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0, https_connection_factory=None): + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, debug, + https_connection_factory) + self.query_conn = None + + def make_request(self, method, path, headers=None, data=''): + # add auth header + if headers == None: + headers = {} + + if not headers.has_key('AWS-Version'): + headers['AWS-Version'] = self.APIVersion + + if not headers.has_key('Content-Type'): + headers['Content-Type'] = self.DefaultContentType + + return AWSAuthConnection.make_request(self, method, path, + headers, data) + + def get_query_connection(self): + if not self.query_conn: + self.query_conn = SQSQueryConnection(self.aws_access_key_id, + self.aws_secret_access_key, + self.is_secure, self.port, + self.proxy, self.proxy_port, + self.server, self.debug, + self.https_connection_factory) + return self.query_conn + + def get_all_queues(self, prefix=''): + if prefix: + path = '/?QueueNamePrefix=%s' % prefix + else: + path = '/' + response = self.make_request('GET', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet([('QueueUrl', Queue)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def get_queue(self, queue_name): + i = 0 + rs = self.get_all_queues(queue_name) + for q in rs: + i += 1 + if i != 1: + return None + else: + return q + + def get_queue_attributes(self, queue_url, attribute='All'): + """ + Performs a GetQueueAttributes request and returns an Attributes + instance (subclass of a Dictionary) holding the requested + attribute name/value pairs. + Inputs: + queue_url - the URL of the desired SQS queue + attribute - All|ApproximateNumberOfMessages|VisibilityTimeout + Default value is "All" + Returns: + An Attribute object which is a mapping type holding the + requested name/value pairs + """ + qc = self.get_query_connection() + return qc.get_queue_attributes(queue_url, attribute) + + def set_queue_attribute(self, queue_url, attribute, value): + """ + Performs a SetQueueAttributes request. + Inputs: + queue_url - The URL of the desired SQS queue + attribute - The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + value - The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + Returns: + Boolean True if successful, otherwise False. + """ + qc = self.get_query_connection() + return qc.set_queue_attribute(queue_url, attribute, value) + + def change_message_visibility(self, queue_url, message_id, vtimeout): + """ + Change the VisibilityTimeout for an individual message. + Inputs: + queue_url - The URL of the desired SQS queue + message_id - The ID of the message whose timeout will be changed + vtimeout - The new VisibilityTimeout value, in seconds + Returns: + Boolean True if successful, otherwise False + Note: This functionality is also available as a method of the + Message object. + """ + qc = self.get_query_connection() + return qc.change_message_visibility(queue_url, message_id, vtimeout) + + def add_grant(self, queue_url, permission, email_address=None, user_id=None): + """ + Add a grant to a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission being granted. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + qc = self.get_query_connection() + return qc.add_grant(queue_url, permission, email_address, user_id) + + def remove_grant(self, queue_url, permission, email_address=None, user_id=None): + """ + Remove a grant from a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission being removed. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + qc = self.get_query_connection() + return qc.remove_grant(queue_url, permission, email_address, user_id) + + def list_grants(self, queue_url, permission=None, email_address=None, user_id=None): + """ + List the grants to a queue. + Inputs: + queue_url - The URL of the desired SQS queue + permission - The permission granted. One of "ReceiveMessage", "SendMessage" or "FullControl". + If supplied, only grants that allow this permission will be returned. + email_address - the email address of the grantee. If supplied, only grants related to this email + address will be returned + user_id - The ID of the grantee. If supplied, only grants related to his user_id will be returned. + Returns: + A string containing the XML Response elements describing the grants. + """ + qc = self.get_query_connection() + return qc.list_grants(queue_url, permission, email_address, user_id) + + def create_queue(self, queue_name, visibility_timeout=None): + """ + Create a new queue. + Inputs: + queue_name - The name of the new queue + visibility_timeout - (Optional) The default visibility + timeout for the new queue. + Returns: + A new Queue object representing the newly created queue. + """ + path = '/?QueueName=%s' % queue_name + if visibility_timeout: + path = path + '&DefaultVisibilityTimeout=%d' % visibility_timeout + response = self.make_request('POST', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + q = Queue(self) + h = handler.XmlHandler(q, self) + xml.sax.parseString(body, h) + return q + + def delete_queue(self, queue, force_deletion=False): + """ + Delete an SQS Queue. + Inputs: + queue - a Queue object representing the SQS queue to be deleted. + force_deletion - (Optional) Normally, SQS will not delete a + queue that contains messages. However, if + the force_deletion argument is True, the + queue will be deleted regardless of whether + there are messages in the queue or not. + USE WITH CAUTION. This will delete all + messages in the queue as well. + Returns: + An empty ResultSet object. Not sure why, actually. It + should probably return a Boolean indicating success or + failure. + """ + method = 'DELETE' + path = queue.id + if force_deletion: + path = path + '?ForceDeletion=true' + response = self.make_request(method, path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + diff --git a/thirdparty/boto/sqs/20070501/message.py b/thirdparty/boto/sqs/20070501/message.py new file mode 100644 index 000000000..0c45b31cc --- /dev/null +++ b/thirdparty/boto/sqs/20070501/message.py @@ -0,0 +1,180 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Message +""" + +import base64 +import StringIO + +class RawMessage: + """ + Base class for SQS messages. RawMessage does not encode the message + in any way. Whatever you store in the body of the message is what + will be written to SQS and whatever is returned from SQS is stored + directly into the body of the message. + """ + + def __init__(self, queue=None, body=''): + self.queue = queue + self._body = '' + self.set_body(body) + self.id = None + + def __len__(self): + return len(self._body) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'MessageBody': + self.set_body(value) + elif name == 'MessageId': + self.id = value + else: + setattr(self, name, value) + + def set_body(self, body): + """ + Set the body of the message. You should always call this method + rather than setting the attribute directly. + """ + self._body = body + + def get_body(self): + """ + Retrieve the body of the message. + """ + return self._body + + def get_body_encoded(self): + """ + This method is really a semi-private method used by the Queue.write + method when writing the contents of the message to SQS. The + RawMessage class does not encode the message in any way so this + just calls get_body(). You probably shouldn't need to call this + method in the normal course of events. + """ + return self.get_body() + + def change_visibility(self, vtimeout): + """ + Convenience function to allow you to directly change the + invisibility timeout for an individual message that has been + read from an SQS queue. This won't affect the default visibility + timeout of the queue. + """ + return self.queue.connection.change_message_visibility(self.queue.id, + self.id, + vtimeout) +class Message(RawMessage): + """ + The default Message class used for SQS queues. This class automatically + encodes/decodes the message body using Base64 encoding to avoid any + illegal characters in the message body. See: + + http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 + + for details on why this is a good idea. The encode/decode is meant to + be transparent to the end-user. + """ + + def endElement(self, name, value, connection): + if name == 'MessageBody': + # Decode the message body returned from SQS using base64 + self.set_body(base64.b64decode(value)) + elif name == 'MessageId': + self.id = value + else: + setattr(self, name, value) + + def get_body_encoded(self): + """ + Because the Message class encodes the message body in base64 + this private method used by queue.write needs to perform the + encoding. + """ + return base64.b64encode(self.get_body()) + +class MHMessage(Message): + """ + The MHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + The encoding/decoding of this is handled automatically and after + the message body has been read, the message instance can be treated + like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def __init__(self, queue=None, body='', xml_attrs=None): + self._dict = {} + Message.__init__(self, queue, body) + + def set_body(self, body): + fp = StringIO.StringIO(body) + line = fp.readline() + while line: + delim = line.find(':') + key = line[0:delim] + value = line[delim+1:].strip() + self._dict[key.strip()] = value.strip() + line = fp.readline() + + def get_body(self): + s = '' + for key,value in self._dict.items(): + s = s + '%s: %s\n' % (key, value) + return s + + def __len__(self): + return len(self.get_body()) + + def __getitem__(self, key): + if self._dict.has_key(key): + return self._dict[key] + else: + raise KeyError(key) + + def __setitem__(self, key, value): + self._dict[key] = value + + def keys(self): + return self._dict.keys() + + def values(self): + return self._dict.values() + + def items(self): + return self._dict.items() + + def has_key(self, key): + return self._dict.has_key(key) + + def update(self, d): + return self._dict.update(d) + + def get(self, key, default=None): + return self._dict.get(key, default) + diff --git a/thirdparty/boto/sqs/20070501/queue.py b/thirdparty/boto/sqs/20070501/queue.py new file mode 100644 index 000000000..b62087755 --- /dev/null +++ b/thirdparty/boto/sqs/20070501/queue.py @@ -0,0 +1,343 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Queue +""" + +import xml.sax +import urlparse +from boto.exception import SQSError +from boto.handler import XmlHandler +from boto.sqs.message import Message +from boto.resultset import ResultSet + +class Queue: + + def __init__(self, connection=None, url=None, message_class=Message): + self.connection = connection + self.url = url + self.message_class = message_class + self.visibility_timeout = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'QueueUrl': + self.url = value + if value: + self.id = urlparse.urlparse(value)[2] + elif name == 'VisibilityTimeout': + self.visibility_timeout = int(value) + else: + setattr(self, name, value) + + def set_message_class(self, message_class): + """ + Set the message class that should be used when instantiating messages read + from the queue. By default, the class boto.sqs.message.Message is used but + this can be overriden with any class that behaves like a message. + Inputs: + message_class - The new message class + Returns: + Nothing + """ + self.message_class = message_class + + def get_attributes(self, attributes='All'): + """ + Retrieves attributes about this queue object and returns + them in an Attribute instance (subclass of a Dictionary). + Inputs: + attributes - A string containing + All|ApproximateNumberOfMessages|VisibilityTimeout + Default value is "All" + Returns: + An Attribute object which is a mapping type holding the + requested name/value pairs + """ + return self.connection.get_queue_attributes(self.id, attributes) + + def set_attribute(self, attribute, value): + """ + Set a new value for an attribute of the Queue. + Inputs: + attribute - The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + value - The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + Returns: + Boolean True if successful, otherwise False. + """ + return self.connection.set_queue_attribute(self.id, attribute, value) + + def add_grant(self, permission, email_address=None, user_id=None): + """ + Add a grant to this queue. + Inputs: + permission - The permission being granted. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + return self.connection.add_grant(self.id, permission, email_address, user_id) + + def remove_grant(self, permission, email_address=None, user_id=None): + """ + Remove a grant from this queue. + Inputs: + permission - The permission being removed. One of "ReceiveMessage", "SendMessage" or "FullControl" + email_address - the email address of the grantee. If email_address is supplied, user_id should be None + user_id - The ID of the grantee. If user_id is supplied, email_address should be None + Returns: + Boolean True if successful, otherwise False + """ + return self.connection.remove_grant(self.id, permission, email_address, user_id) + + def list_grants(self, permission=None, email_address=None, user_id=None): + """ + List the grants to this queue. + Inputs: + permission - The permission granted. One of "ReceiveMessage", "SendMessage" or "FullControl". + If supplied, only grants that allow this permission will be returned. + email_address - the email address of the grantee. If supplied, only grants related to this email + address will be returned + user_id - The ID of the grantee. If supplied, only grants related to his user_id will be returned. + Returns: + A string containing the XML Response elements describing the grants. + """ + return self.connection.list_grants(self.id, permission, email_address, user_id) + + def get_timeout(self): + """ + Get the visibility timeout for the queue. + Inputs: + None + Returns: + The number of seconds as an integer. + """ + a = self.get_attributes('VisibilityTimeout') + return int(a['VisibilityTimeout']) + + def set_timeout(self, visibility_timeout): + """ + Set the visibility timeout for the queue. + Inputs: + visibility_timeout - The desired timeout in seconds + Returns: + Nothing + """ + retval = self.set_attribute('VisibilityTimeout', visibility_timeout) + if retval: + self.visibility_timeout = visibility_timeout + return retval + + def read(self, visibility_timeout=None): + """ + Read a single message from the queue. + Inputs: + visibility_timeout - The timeout for this message in seconds + Returns: + A single message or None if queue is empty + """ + rs = self.get_messages(1, visibility_timeout) + if len(rs) == 1: + return rs[0] + else: + return None + + def write(self, message): + """ + Add a single message to the queue. + Inputs: + message - The message to be written to the queue + Returns: + None + """ + path = '%s/back' % self.id + message.queue = self + response = self.connection.make_request('PUT', path, None, + message.get_body_encoded()) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + handler = XmlHandler(message, self.connection) + xml.sax.parseString(body, handler) + return None + + def new_message(self, body=''): + return self.message_class(self, body) + + # get a variable number of messages, returns a list of messages + def get_messages(self, num_messages=1, visibility_timeout=None): + path = '%s/front?NumberOfMessages=%d' % (self.id, num_messages) + if visibility_timeout: + path = '%s&VisibilityTimeout=%d' % (path, visibility_timeout) + response = self.connection.make_request('GET', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet([('Message', self.message_class)]) + h = XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def delete_message(self, message): + path = '%s/%s' % (self.id, message.id) + response = self.connection.make_request('DELETE', path) + body = response.read() + if response.status >= 300: + raise SQSError(response.status, response.reason, body) + rs = ResultSet() + h = XmlHandler(rs, self.connection) + xml.sax.parseString(body, h) + return rs + + def clear(self, page_size=100, vtimeout=10): + """Utility function to remove all messages from a queue""" + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + self.delete_message(m) + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def count(self, page_size=100, vtimeout=10): + """ + Utility function to count the number of messages in a queue. + Note: This function now calls GetQueueAttributes to obtain + an 'approximate' count of the number of messages in a queue. + """ + a = self.get_attributes('ApproximateNumberOfMessages') + return a['ApproximateNumberOfMessages'] + + def count_slow(self, page_size=100, vtimeout=10): + """ + Deprecated. This is the old 'count' method that actually counts + the messages by reading them all. This gives an accurate count but + is very slow for queues with non-trivial number of messasges. + Instead, use get_attribute('ApproximateNumberOfMessages') to take + advantage of the new SQS capability. This is retained only for + the unit tests. + """ + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def dump(self, file_name, page_size=100, vtimeout=10, sep='\n'): + """Utility function to dump the messages in a queue to a file""" + fp = open(file_name, 'wb') + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + fp.write(m.get_body()) + if sep: + fp.write(sep) + n += 1 + l = self.get_messages(page_size, vtimeout) + fp.close() + return n + + def save(self, file_name, sep='\n'): + """ + Read all messages from the queue and persist them to local file. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + fp = open(file_name, 'wb') + n = 0 + m = self.read() + while m: + n += 1 + fp.write(m.get_body()) + if sep: + fp.write(sep) + self.delete_message(m) + m = self.read() + fp.close() + return n + + def save_to_s3(self, bucket): + """ + Read all messages from the queue and persist them to S3. + Messages are stored in the S3 bucket using a naming scheme of: + / + Messages are deleted from the queue after being saved to S3. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + key = bucket.new_key('%s/%s' % (self.id, m.id)) + key.set_contents_from_string(m.get_body()) + self.delete_message(m) + m = self.read() + return n + + def load_from_s3(self, bucket, prefix=None): + """ + Load messages previously saved to S3. + """ + n = 0 + if prefix: + prefix = '%s/' % prefix + else: + prefix = '%s/' % self.id + rs = bucket.list(prefix=prefix) + for key in rs: + n += 1 + m = self.new_message(key.get_contents_as_string()) + self.write(m) + return n + + def load(self, file_name, sep='\n'): + """Utility function to load messages from a file to a queue""" + fp = open(file_name, 'rb') + n = 0 + body = '' + l = fp.readline() + while l: + if l == sep: + m = Message(self, body) + self.write(m) + n += 1 + print 'writing message %d' % n + body = '' + else: + body = body + l + l = fp.readline() + fp.close() + return n + + diff --git a/thirdparty/boto/sqs/20070501/readme.txt b/thirdparty/boto/sqs/20070501/readme.txt new file mode 100644 index 000000000..71325797f --- /dev/null +++ b/thirdparty/boto/sqs/20070501/readme.txt @@ -0,0 +1,6 @@ +The main SQS implementation now uses the 2008-01-01 API verson. To use the older API version +(2007-05-01) you need to edit your /etc/boto.cfg or ~/.boto file to add the following line: + +boto.sqs_extend = 20070501 + +This will allow the code in the boto.sqs.20070501 module to override the code in boto.sqs. diff --git a/thirdparty/boto/sqs/__init__.py b/thirdparty/boto/sqs/__init__.py new file mode 100644 index 000000000..5be8e3e47 --- /dev/null +++ b/thirdparty/boto/sqs/__init__.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto + +boto.check_extensions(__name__, __path__) + +__all__ = ['Connection', 'Queue', 'Message', 'MHMessage'] + +from connection import SQSConnection as Connection +from queue import Queue +from message import Message, MHMessage diff --git a/thirdparty/boto/sqs/attributes.py b/thirdparty/boto/sqs/attributes.py new file mode 100644 index 000000000..26c720416 --- /dev/null +++ b/thirdparty/boto/sqs/attributes.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Attribute Name/Value set +""" + +class Attributes(dict): + + def __init__(self, parent): + self.parent = parent + self.current_key = None + self.current_value = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Attribute': + self[self.current_key] = self.current_value + elif name == 'Name': + self.current_key = value + elif name == 'Value': + self.current_value = value + else: + setattr(self, name, value) + + diff --git a/thirdparty/boto/sqs/connection.py b/thirdparty/boto/sqs/connection.py new file mode 100644 index 000000000..4de05df27 --- /dev/null +++ b/thirdparty/boto/sqs/connection.py @@ -0,0 +1,129 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSQueryConnection +import xml.sax +from boto.sqs.queue import Queue +from boto.sqs.message import Message +from boto.sqs.attributes import Attributes +from boto import handler +from boto.resultset import ResultSet +from boto.exception import SQSError + +PERM_ReceiveMessage = 'ReceiveMessage' +PERM_SendMessage = 'SendMessage' +PERM_FullControl = 'FullControl' + +AllPermissions = [PERM_ReceiveMessage, PERM_SendMessage, PERM_FullControl] + +class SQSConnection(AWSQueryConnection): + + """ + A subclass of the original SQSQueryConnection targeting the 2008-01-01 SQS API. + """ + + DefaultHost = 'queue.amazonaws.com' + APIVersion = '2008-01-01' + SignatureVersion = '1' + DefaultContentType = 'text/plain' + ResponseError = SQSError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=DefaultHost, debug=0, + https_connection_factory=None): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory) + + def create_queue(self, queue_name, visibility_timeout=None): + params = {'QueueName': queue_name} + if visibility_timeout: + params['DefaultVisibilityTimeout'] = '%d' % (visibility_timeout,) + return self.get_object('CreateQueue', params, Queue) + + def delete_queue(self, queue, force_deletion=False): + """ + Delete an SQS Queue. + + @type queue: A Queue object + @param queue: The SQS queue to be deleted + @type force_deletion: Boolean + @param force_deletion: Normally, SQS will not delete a queue that contains messages. + However, if the force_deletion argument is True, the + queue will be deleted regardless of whether there are messages in + the queue or not. USE WITH CAUTION. This will delete all + messages in the queue as well. + @rtype: bool + @return: True if the command succeeded, False otherwise + """ + return self.get_status('DeleteQueue', None, queue.url) + + def get_queue_attributes(self, queue, attribute='All'): + params = {'AttributeName' : attribute} + return self.get_object('GetQueueAttributes', params, Attributes, queue.url) + + def set_queue_attribute(self, queue, attribute, value): + params = {'Attribute.Name' : attribute, 'Attribute.Value' : value} + return self.get_status('SetQueueAttributes', params, queue.url) + + def receive_message(self, queue, number_messages=1, + visibility_timeout=None): + """ + Read messages from an SQS Queue. + + @type queue: A Queue object or a queue URL. + @param queue: The Queue from which messages are read. + @type number_messages: int + @param number_messages: The maximum number of messages to read (default=1) + @type visibility_timeout: int + @param visibility_timeout: The number of seconds the message should remain invisible + to other queue readers (default=None which uses the Queues default) + + """ + params = {'MaxNumberOfMessages' : number_messages} + if visibility_timeout: + params['VisibilityTimeout'] = visibility_timeout + return self.get_list('ReceiveMessage', params, [('Message', queue.message_class)], + queue.url, queue) + + def delete_message(self, queue, message): + params = {'ReceiptHandle' : message.receipt_handle} + return self.get_status('DeleteMessage', params, queue.url) + + def send_message(self, queue, message_content): + params = {'MessageBody' : message_content} + return self.get_status('SendMessage', params, queue.url) + + def get_all_queues(self, prefix=''): + params = {} + if prefix: + params['QueueNamePrefix'] = prefix + return self.get_list('ListQueues', params, [('QueueUrl', Queue)]) + + def get_queue(self, queue_name): + rs = self.get_all_queues(queue_name) + if len(rs) == 1: + return rs[0] + return None + + lookup = get_queue + diff --git a/thirdparty/boto/sqs/jsonmessage.py b/thirdparty/boto/sqs/jsonmessage.py new file mode 100644 index 000000000..2331519ff --- /dev/null +++ b/thirdparty/boto/sqs/jsonmessage.py @@ -0,0 +1,37 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sqs.message import MHMessage +import base64 +import simplejson + +class JSONMessage(MHMessage): + """ + Acts like a dictionary but encodes it's data as a Base64 encoded JSON payload. + """ + + def decode(self, value): + value = base64.b64decode(value) + return simplejson.loads(value) + + def encode(self, value): + value = simplejson.dumps(value) + return base64.b64encode(value) diff --git a/thirdparty/boto/sqs/message.py b/thirdparty/boto/sqs/message.py new file mode 100644 index 000000000..e95f6a9e5 --- /dev/null +++ b/thirdparty/boto/sqs/message.py @@ -0,0 +1,206 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +SQS Message + +A Message represents the data stored in an SQS queue. The rules for what is allowed within an SQS +Message are here: + + http://docs.amazonwebservices.com/AWSSimpleQueueService/2008-01-01/SQSDeveloperGuide/Query_QuerySendMessage.html + +So, at it's simplest level a Message just needs to allow a developer to store bytes in it and get the bytes +back out. However, to allow messages to have richer semantics, the Message class must support the +following interfaces: + +The constructor for the Message class must accept a keyword parameter "queue" which is an instance of a +boto Queue object and represents the queue that the message will be stored in. The default value for +this parameter is None. + +The constructor for the Message class must accept a keyword parameter "body" which represents the +content or body of the message. The format of this parameter will depend on the behavior of the +particular Message subclass. For example, if the Message subclass provides dictionary-like behavior to the +user the body passed to the constructor should be a dict-like object that can be used to populate +the initial state of the message. + +The Message class must provide an encode method that accepts a value of the same type as the body +parameter of the constructor and returns a string of characters that are able to be stored in an +SQS message body (see rules above). + +The Message class must provide a decode method that accepts a string of characters that can be +stored (and probably were stored!) in an SQS message and return an object of a type that is consistent +with the "body" parameter accepted on the class constructor. + +The Message class must provide a __len__ method that will return the size of the encoded message +that would be stored in SQS based on the current state of the Message object. + +The Message class must provide a get_body method that will return the body of the message in the +same format accepted in the constructor of the class. + +The Message class must provide a set_body method that accepts a message body in the same format +accepted by the constructor of the class. This method should alter to the internal state of the +Message object to reflect the state represented in the message body parameter. + +The Message class must provide a get_body_encoded method that returns the current body of the message +in the format in which it would be stored in SQS. +""" + +import base64 +import StringIO + +class RawMessage: + """ + Base class for SQS messages. RawMessage does not encode the message + in any way. Whatever you store in the body of the message is what + will be written to SQS and whatever is returned from SQS is stored + directly into the body of the message. + """ + + def __init__(self, queue=None, body=''): + self.queue = queue + self.set_body(body) + self.id = None + self.receipt_handle = None + + def __len__(self): + return len(self.encode(self._body)) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Body': + self.set_body(self.decode(value)) + elif name == 'MessageId': + self.id = value + elif name == 'ReceiptHandle': + self.receipt_handle = value + else: + setattr(self, name, value) + + def encode(self, value): + """Transform body object into serialized byte array format.""" + return value + + def decode(self, value): + """Transform seralized byte array into any object.""" + return value + + def set_body(self, body): + """Override the current body for this object, using decoded format.""" + self._body = body + + def get_body(self): + return self._body + + def get_body_encoded(self): + """ + This method is really a semi-private method used by the Queue.write + method when writing the contents of the message to SQS. + You probably shouldn't need to call this method in the normal course of events. + """ + return self.encode(self.get_body()) + + def delete(self): + if self.queue: + return self.queue.delete_message(self) + +class Message(RawMessage): + """ + The default Message class used for SQS queues. This class automatically + encodes/decodes the message body using Base64 encoding to avoid any + illegal characters in the message body. See: + + http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 + + for details on why this is a good idea. The encode/decode is meant to + be transparent to the end-user. + """ + + def encode(self, value): + return base64.b64encode(value) + + def decode(self, value): + return base64.b64decode(value) + +class MHMessage(Message): + """ + The MHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + The encoding/decoding of this is handled automatically and after + the message body has been read, the message instance can be treated + like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def __init__(self, queue=None, body=None, xml_attrs=None): + if body == None or body == '': + body = {} + Message.__init__(self, queue, body) + + def decode(self, value): + msg = {} + fp = StringIO.StringIO(value) + line = fp.readline() + while line: + delim = line.find(':') + key = line[0:delim] + value = line[delim+1:].strip() + msg[key.strip()] = value.strip() + line = fp.readline() + return msg + + def encode(self, value): + s = '' + for item in value.items(): + s = s + '%s: %s\n' % (item[0], item[1]) + return s + + def __getitem__(self, key): + if self._body.has_key(key): + return self._body[key] + else: + raise KeyError(key) + + def __setitem__(self, key, value): + self._body[key] = value + self.set_body(self._body) + + def keys(self): + return self._body.keys() + + def values(self): + return self._body.values() + + def items(self): + return self._body.items() + + def has_key(self, key): + return self._body.has_key(key) + + def update(self, d): + self._body.update(d) + self.set_body(self._body) + + def get(self, key, default=None): + return self._body.get(key, default) diff --git a/thirdparty/boto/sqs/queue.py b/thirdparty/boto/sqs/queue.py new file mode 100644 index 000000000..344978563 --- /dev/null +++ b/thirdparty/boto/sqs/queue.py @@ -0,0 +1,316 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Queue +""" + +import xml.sax +import urlparse +from boto.exception import SQSError +from boto.handler import XmlHandler +from boto.sqs.message import Message +from boto.resultset import ResultSet + +class Queue: + + def __init__(self, connection=None, url=None, message_class=Message): + self.connection = connection + self.url = url + self.message_class = message_class + self.visibility_timeout = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'QueueUrl': + self.url = value + if value: + self.id = urlparse.urlparse(value)[2] + elif name == 'VisibilityTimeout': + self.visibility_timeout = int(value) + else: + setattr(self, name, value) + + def set_message_class(self, message_class): + """ + Set the message class that should be used when instantiating messages read + from the queue. By default, the class boto.sqs.message.Message is used but + this can be overriden with any class that behaves like a message. + + @type message_class: Message-like class + @param message_class: The new Message class + """ + self.message_class = message_class + + def get_attributes(self, attributes='All'): + """ + Retrieves attributes about this queue object and returns + them in an Attribute instance (subclass of a Dictionary). + + @type attributes: string + @param attributes: String containing one of + All|ApproximateNumberOfMessages|VisibilityTimeout + Default value is "All" + @rtype: Attribute object + @return: An Attribute object which is a mapping type holding the + requested name/value pairs + """ + return self.connection.get_queue_attributes(self, attributes) + + def set_attribute(self, attribute, value): + """ + Set a new value for an attribute of the Queue. + + @type attribute: String + @param attribute: The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + @type value: int + @param value: The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + + @rtype: bool + @return: True if successful, otherwise False. + """ + return self.connection.set_queue_attribute(self, attribute, value) + + def get_timeout(self): + """ + Get the visibility timeout for the queue. + + @rtype: int + @return: The number of seconds as an integer. + """ + a = self.get_attributes('VisibilityTimeout') + return int(a['VisibilityTimeout']) + + def set_timeout(self, visibility_timeout): + """ + Set the visibility timeout for the queue. + + @type visibility_timeout: int + @param visibility_timeout: The desired timeout in seconds + """ + retval = self.set_attribute('VisibilityTimeout', visibility_timeout) + if retval: + self.visibility_timeout = visibility_timeout + return retval + + def read(self, visibility_timeout=None): + """ + Read a single message from the queue. + + @type visibility_timeout: int + @param visibility_timeout: The timeout for this message in seconds + + @rtype: Message + @return: A single message or None if queue is empty + """ + rs = self.get_messages(1, visibility_timeout) + if len(rs) == 1: + return rs[0] + else: + return None + + def write(self, message): + """ + Add a single message to the queue. + + @type message: Message + @param message: The message to be written to the queue + """ + self.connection.send_message(self, message.get_body_encoded()) + return None + + def new_message(self, body=''): + """ + Create new message of appropriate class. + + @type body: message body + @param body: The body of the newly created message (optional). + """ + m = self.message_class(self, body) + m.queue = self + return m + + # get a variable number of messages, returns a list of messages + def get_messages(self, num_messages=1, visibility_timeout=None): + """ + Get a variable number of messages. + + @type num_messages: int + @param num_messages: The maximum number of messages to read from the queue. + @type visibility_timeout: int + @param visibility_timeout: The VisibilityTimeout for the messages read. + + @rtype: list + @return: A list of messages. + """ + return self.connection.receive_message(self, number_messages=num_messages, + visibility_timeout=visibility_timeout) + + def delete_message(self, message): + """ + Delete a message from the queue. + + @type message: Message + @param message: The message object to delete. + + @rtype: bool + @return: True if successful, False otherwise + """ + return self.connection.delete_message(self, message) + + def delete(self): + """ + Delete the queue. + """ + return self.connection.delete_queue(self) + + def clear(self, page_size=10, vtimeout=10): + """Utility function to remove all messages from a queue""" + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + self.delete_message(m) + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def count(self, page_size=10, vtimeout=10): + """ + Utility function to count the number of messages in a queue. + Note: This function now calls GetQueueAttributes to obtain + an 'approximate' count of the number of messages in a queue. + """ + a = self.get_attributes('ApproximateNumberOfMessages') + return a['ApproximateNumberOfMessages'] + + def count_slow(self, page_size=10, vtimeout=10): + """ + Deprecated. This is the old 'count' method that actually counts + the messages by reading them all. This gives an accurate count but + is very slow for queues with non-trivial number of messasges. + Instead, use get_attribute('ApproximateNumberOfMessages') to take + advantage of the new SQS capability. This is retained only for + the unit tests. + """ + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def dump(self, file_name, page_size=100, vtimeout=10, sep='\n'): + """Utility function to dump the messages in a queue to a file""" + fp = open(file_name, 'wb') + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + fp.write(m.get_body()) + if sep: + fp.write(sep) + n += 1 + l = self.get_messages(page_size, vtimeout) + fp.close() + return n + + def save(self, file_name, sep='\n'): + """ + Read all messages from the queue and persist them to local file. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + fp = open(file_name, 'wb') + n = 0 + m = self.read() + while m: + n += 1 + fp.write(m.get_body()) + if sep: + fp.write(sep) + self.delete_message(m) + m = self.read() + fp.close() + return n + + def save_to_s3(self, bucket): + """ + Read all messages from the queue and persist them to S3. + Messages are stored in the S3 bucket using a naming scheme of: + / + Messages are deleted from the queue after being saved to S3. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + key = bucket.new_key('%s/%s' % (self.id, m.id)) + key.set_contents_from_string(m.get_body()) + self.delete_message(m) + m = self.read() + return n + + def load_from_s3(self, bucket, prefix=None): + """ + Load messages previously saved to S3. + """ + n = 0 + if prefix: + prefix = '%s/' % prefix + else: + prefix = '%s/' % self.id + rs = bucket.list(prefix=prefix) + for key in rs: + n += 1 + m = self.new_message(key.get_contents_as_string()) + self.write(m) + return n + + def load(self, file_name, sep='\n'): + """Utility function to load messages from a file to a queue""" + fp = open(file_name, 'rb') + n = 0 + body = '' + l = fp.readline() + while l: + if l == sep: + m = Message(self, body) + self.write(m) + n += 1 + print 'writing message %d' % n + body = '' + else: + body = body + l + l = fp.readline() + fp.close() + return n + + diff --git a/thirdparty/boto/tests/__init__.py b/thirdparty/boto/tests/__init__.py new file mode 100644 index 000000000..449bd162a --- /dev/null +++ b/thirdparty/boto/tests/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/thirdparty/boto/tests/test.py b/thirdparty/boto/tests/test.py new file mode 100644 index 000000000..c6175cac0 --- /dev/null +++ b/thirdparty/boto/tests/test.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +do the unit tests! +""" + +import sys, os, unittest +import getopt, sys +import boto + +from boto.tests.test_sqsconnection import SQSConnectionTest +from boto.tests.test_s3connection import S3ConnectionTest +from boto.tests.test_ec2connection import EC2ConnectionTest +from boto.tests.test_sdbconnection import SDBConnectionTest + +def usage(): + print 'test.py [-t testsuite] [-v verbosity]' + print ' -t run specific testsuite (s3|sqs|ec2|sdb|all)' + print ' -v verbosity (0|1|2)' + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'ht:v:', + ['help', 'testsuite', 'verbosity']) + except: + usage() + sys.exit(2) + testsuite = 'all' + verbosity = 1 + for o, a in opts: + if o in ('-h', '--help'): + usage() + sys.exit() + if o in ('-t', '--testsuite'): + testsuite = a + if o in ('-v', '--verbosity'): + verbosity = int(a) + if len(args) != 0: + usage() + sys.exit() + suite = unittest.TestSuite() + if testsuite == 'all': + suite.addTest(unittest.makeSuite(SQSConnectionTest)) + suite.addTest(unittest.makeSuite(S3ConnectionTest)) + suite.addTest(unittest.makeSuite(EC2ConnectionTest)) + suite.addTest(unittest.makeSuite(SDBConnectionTest)) + elif testsuite == 's3': + suite.addTest(unittest.makeSuite(S3ConnectionTest)) + elif testsuite == 'sqs': + suite.addTest(unittest.makeSuite(SQSConnectionTest)) + elif testsuite == 'ec2': + suite.addTest(unittest.makeSuite(EC2ConnectionTest)) + elif testsuite == 'sdb': + suite.addTest(unittest.makeSuite(SDBConnectionTest)) + else: + usage() + sys.exit() + unittest.TextTestRunner(verbosity=verbosity).run(suite) + +if __name__ == "__main__": + main() diff --git a/thirdparty/boto/tests/test_ec2connection.py b/thirdparty/boto/tests/test_ec2connection.py new file mode 100644 index 000000000..8f1fb5985 --- /dev/null +++ b/thirdparty/boto/tests/test_ec2connection.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the EC2Connection +""" + +import unittest +import time +import os +from boto.ec2.connection import EC2Connection +import telnetlib +import socket + +class EC2ConnectionTest (unittest.TestCase): + + def test_1_basic(self): + # this is my user_id, if you want to run these tests you should + # replace this with yours or they won't work + user_id = '084307701560' + print '--- running EC2Connection tests ---' + c = EC2Connection() + # get list of private AMI's + rs = c.get_all_images(owners=[user_id]) + assert len(rs) > 0 + # now pick the first one + image = rs[0] + # temporarily make this image runnable by everyone + status = image.set_launch_permissions(group_names=['all']) + assert status + d = image.get_launch_permissions() + assert d.has_key('groups') + assert len(d['groups']) > 0 + # now remove that permission + status = image.remove_launch_permissions(group_names=['all']) + assert status + d = image.get_launch_permissions() + assert not d.has_key('groups') + + # create a new security group + group_name = 'test-%d' % int(time.time()) + group_desc = 'This is a security group created during unit testing' + group = c.create_security_group(group_name, group_desc) + # now get a listing of all security groups and look for our new one + rs = c.get_all_security_groups() + found = False + for g in rs: + if g.name == group_name: + found = True + assert found + # now pass arg to filter results to only our new group + rs = c.get_all_security_groups([group_name]) + assert len(rs) == 1 + group = rs[0] + # + # now delete the security group + status = c.delete_security_group(group_name) + # now make sure it's really gone + rs = c.get_all_security_groups() + found = False + for g in rs: + if g.name == group_name: + found = True + assert not found + # now create it again for use with the instance test + group = c.create_security_group(group_name, group_desc) + + # now try to launch apache image with our new security group + rs = c.get_all_images() + img_loc = 'ec2-public-images/fedora-core4-apache.manifest.xml' + for image in rs: + if image.location == img_loc: + break + reservation = image.run(security_groups=[group.name]) + instance = reservation.instances[0] + while instance.state != 'running': + print '\tinstance is %s' % instance.state + time.sleep(30) + instance.update() + # instance in now running, try to telnet to port 80 + t = telnetlib.Telnet() + try: + t.open(instance.dns_name, 80) + except socket.error: + pass + # now open up port 80 and try again, it should work + group.authorize('tcp', 80, 80, '0.0.0.0/0') + t.open(instance.dns_name, 80) + t.close() + # now revoke authorization and try again + group.revoke('tcp', 80, 80, '0.0.0.0/0') + try: + t.open(instance.dns_name, 80) + except socket.error: + pass + # now kill the instance and delete the security group + instance.stop() + # unfortunately, I can't delete the sg within this script + #sg.delete() + + # create a new key pair + key_name = 'test-%d' % int(time.time()) + status = c.create_key_pair(key_name) + assert status + # now get a listing of all key pairs and look for our new one + rs = c.get_all_key_pairs() + found = False + for k in rs: + if k.name == key_name: + found = True + assert found + # now pass arg to filter results to only our new key pair + rs = c.get_all_key_pairs([key_name]) + assert len(rs) == 1 + key_pair = rs[0] + # now delete the key pair + status = c.delete_key_pair(key_name) + # now make sure it's really gone + rs = c.get_all_key_pairs() + found = False + for k in rs: + if k.name == key_name: + found = True + assert not found + + # short test around Paid AMI capability + demo_paid_ami_id = 'ami-bd9d78d4' + demo_paid_ami_product_code = 'A79EC0DB' + l = c.get_all_images([demo_paid_ami_id]) + assert len(l) == 1 + assert len(l[0].product_codes) == 1 + assert l[0].product_codes[0] == demo_paid_ami_product_code + + print '--- tests completed ---' diff --git a/thirdparty/boto/tests/test_s3connection.py b/thirdparty/boto/tests/test_s3connection.py new file mode 100644 index 000000000..ecf7ebfd0 --- /dev/null +++ b/thirdparty/boto/tests/test_s3connection.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the S3Connection +""" + +import unittest +import time +import os +from boto.s3.connection import S3Connection +from boto.exception import S3PermissionsError + +class S3ConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running S3Connection tests ---' + c = S3Connection() + # create a new, empty bucket + bucket_name = 'test-%d' % int(time.time()) + bucket = c.create_bucket(bucket_name) + # now try a get_bucket call and see if it's really there + bucket = c.get_bucket(bucket_name) + # create a new key and store it's content from a string + k = bucket.new_key() + k.name = 'foobar' + s1 = 'This is a test of file upload and download' + s2 = 'This is a second string to test file upload and download' + k.set_contents_from_string(s1) + fp = open('foobar', 'wb') + # now get the contents from s3 to a local file + k.get_contents_to_file(fp) + fp.close() + fp = open('foobar') + # check to make sure content read from s3 is identical to original + assert s1 == fp.read(), 'corrupted file' + fp.close() + bucket.delete_key(k) + # test a few variations on get_all_keys - first load some data + # for the first one, let's override the content type + phony_mimetype = 'application/x-boto-test' + headers = {'Content-Type': phony_mimetype} + k.name = 'foo/bar' + k.set_contents_from_string(s1, headers) + k.name = 'foo/bas' + k.set_contents_from_filename('foobar') + k.name = 'foo/bat' + k.set_contents_from_string(s1) + k.name = 'fie/bar' + k.set_contents_from_string(s1) + k.name = 'fie/bas' + k.set_contents_from_string(s1) + k.name = 'fie/bat' + k.set_contents_from_string(s1) + # try resetting the contents to another value + md5 = k.md5 + k.set_contents_from_string(s2) + assert k.md5 != md5 + os.unlink('foobar') + all = bucket.get_all_keys() + assert len(all) == 6 + rs = bucket.get_all_keys(prefix='foo') + assert len(rs) == 3 + rs = bucket.get_all_keys(prefix='', delimiter='/') + assert len(rs) == 2 + rs = bucket.get_all_keys(maxkeys=5) + assert len(rs) == 5 + # test the lookup method + k = bucket.lookup('foo/bar') + assert isinstance(k, bucket.key_class) + assert k.content_type == phony_mimetype + k = bucket.lookup('notthere') + assert k == None + # try some metadata stuff + k = bucket.new_key() + k.name = 'has_metadata' + mdkey1 = 'meta1' + mdval1 = 'This is the first metadata value' + k.set_metadata(mdkey1, mdval1) + mdkey2 = 'meta2' + mdval2 = 'This is the second metadata value' + k.set_metadata(mdkey2, mdval2) + k.set_contents_from_string(s1) + k = bucket.lookup('has_metadata') + assert k.get_metadata(mdkey1) == mdval1 + assert k.get_metadata(mdkey2) == mdval2 + k = bucket.new_key() + k.name = 'has_metadata' + k.get_contents_as_string() + assert k.get_metadata(mdkey1) == mdval1 + assert k.get_metadata(mdkey2) == mdval2 + bucket.delete_key(k) + # try a key with a funny character + rs = bucket.get_all_keys() + num_keys = len(rs) + k = bucket.new_key() + k.name = 'testnewline\n' + k.set_contents_from_string('This is a test') + rs = bucket.get_all_keys() + assert len(rs) == num_keys + 1 + bucket.delete_key(k) + rs = bucket.get_all_keys() + assert len(rs) == num_keys + # try some acl stuff + bucket.set_acl('public-read') + policy = bucket.get_acl() + assert len(policy.acl.grants) == 2 + bucket.set_acl('private') + policy = bucket.get_acl() + assert len(policy.acl.grants) == 1 + k = bucket.lookup('foo/bar') + k.set_acl('public-read') + policy = k.get_acl() + assert len(policy.acl.grants) == 2 + k.set_acl('private') + policy = k.get_acl() + assert len(policy.acl.grants) == 1 + # try the convenience methods for grants + bucket.add_user_grant('FULL_CONTROL', + 'c1e724fbfa0979a4448393c59a8c055011f739b6d102fb37a65f26414653cd67') + try: + bucket.add_email_grant('foobar', 'foo@bar.com') + except S3PermissionsError: + pass + # now delete all keys in bucket + for k in all: + bucket.delete_key(k) + # now delete bucket + c.delete_bucket(bucket) + print '--- tests completed ---' diff --git a/thirdparty/boto/tests/test_sdbconnection.py b/thirdparty/boto/tests/test_sdbconnection.py new file mode 100644 index 000000000..1dc16652b --- /dev/null +++ b/thirdparty/boto/tests/test_sdbconnection.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the SDBConnection +""" + +import unittest +import time +from boto.sdb.connection import SDBConnection +from boto.exception import SDBResponseError + +class SDBConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running SDBConnection tests ---' + c = SDBConnection() + rs = c.get_all_domains() + num_domains = len(rs) + + # try illegal name + try: + domain = c.create_domain('bad:domain:name') + except SDBResponseError: + pass + + # now create one that should work and should be unique (i.e. a new one) + domain_name = 'test%d' % int(time.time()) + domain = c.create_domain(domain_name) + rs = c.get_all_domains() + assert len(rs) == num_domains+1 + + # now let's a couple of items and attributes + item_1 = 'item1' + same_value = 'same_value' + attrs_1 = {'name1' : same_value, 'name2' : 'diff_value_1'} + domain.put_attributes(item_1, attrs_1) + item_2 = 'item2' + attrs_2 = {'name1' : same_value, 'name2' : 'diff_value_2'} + domain.put_attributes(item_2, attrs_2) + time.sleep(10) + + # try to get the attributes and see if they match + item = domain.get_attributes(item_1) + assert len(item.keys()) == len(attrs_1.keys()) + assert item['name1'] == attrs_1['name1'] + assert item['name2'] == attrs_1['name2'] + + # try a search or two + rs = domain.query("['name1'='%s']" % same_value) + n = 0 + for item in rs: + n += 1 + assert n == 2 + rs = domain.query("['name2'='diff_value_2']") + n = 0 + for item in rs: + n += 1 + assert n == 1 + + # delete all attributes associated with item_1 + stat = domain.delete_attributes(item_1) + assert stat + + # now delete the domain + stat = c.delete_domain(domain) + assert stat + + print '--- tests completed ---' + diff --git a/thirdparty/boto/tests/test_sqsconnection.py b/thirdparty/boto/tests/test_sqsconnection.py new file mode 100644 index 000000000..802d329f4 --- /dev/null +++ b/thirdparty/boto/tests/test_sqsconnection.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for the SQSConnection +""" + +import unittest +import time +from boto.sqs.connection import SQSConnection +from boto.sqs.message import MHMessage +from boto.exception import SQSError + +class SQSConnectionTest (unittest.TestCase): + + def test_1_basic(self): + print '--- running SQSConnection tests ---' + c = SQSConnection() + rs = c.get_all_queues() + num_queues = 0 + for q in rs: + num_queues += 1 + + # try illegal name + try: + queue = c.create_queue('bad_queue_name') + except SQSError: + pass + + # now create one that should work and should be unique (i.e. a new one) + queue_name = 'test%d' % int(time.time()) + timeout = 60 + queue = c.create_queue(queue_name, timeout) + time.sleep(30) + rs = c.get_all_queues() + i = 0 + for q in rs: + i += 1 + assert i == num_queues+1 + assert queue.count_slow() == 0 + + # check the visibility timeout + t = queue.get_timeout() + assert t == timeout, '%d != %d' % (t, timeout) + + # now try to get queue attributes + a = q.get_attributes() + assert a.has_key('ApproximateNumberOfMessages') + assert a.has_key('VisibilityTimeout') + a = q.get_attributes('ApproximateNumberOfMessages') + assert a.has_key('ApproximateNumberOfMessages') + assert not a.has_key('VisibilityTimeout') + a = q.get_attributes('VisibilityTimeout') + assert not a.has_key('ApproximateNumberOfMessages') + assert a.has_key('VisibilityTimeout') + + # now change the visibility timeout + timeout = 45 + queue.set_timeout(timeout) + time.sleep(30) + t = queue.get_timeout() + assert t == timeout, '%d != %d' % (t, timeout) + + # now add a message + message_body = 'This is a test\n' + message = queue.new_message(message_body) + queue.write(message) + time.sleep(30) + assert queue.count_slow() == 1 + time.sleep(30) + + # now read the message from the queue with a 10 second timeout + message = queue.read(visibility_timeout=10) + assert message + assert message.get_body() == message_body + + # now immediately try another read, shouldn't find anything + message = queue.read() + assert message == None + + # now wait 10 seconds and try again + time.sleep(10) + message = queue.read() + assert message + + if c.APIVersion == '2007-05-01': + # now terminate the visibility timeout for this message + message.change_visibility(0) + # now see if we can read it in the queue + message = queue.read() + assert message + + # now delete the message + queue.delete_message(message) + time.sleep(30) + assert queue.count_slow() == 0 + + # create another queue so we can test force deletion + # we will also test MHMessage with this queue + queue_name = 'test%d' % int(time.time()) + timeout = 60 + queue = c.create_queue(queue_name, timeout) + queue.set_message_class(MHMessage) + time.sleep(30) + + # now add a couple of messages + message = queue.new_message() + message['foo'] = 'bar' + queue.write(message) + message_body = {'fie' : 'baz', 'foo' : 'bar'} + message = queue.new_message(body=message_body) + queue.write(message) + time.sleep(30) + + m = queue.read() + assert m['foo'] == 'bar' + + # now delete that queue and messages + c.delete_queue(queue, True) + + print '--- tests completed ---' + diff --git a/thirdparty/boto/utils.py b/thirdparty/boto/utils.py new file mode 100644 index 000000000..6109be7b9 --- /dev/null +++ b/thirdparty/boto/utils.py @@ -0,0 +1,457 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Some handy utility functions used by several classes. +""" + +import base64 +import hmac +import re +import sha +import urllib, urllib2 +import imp +import popen2, os, StringIO +import time, datetime +import logging.handlers +import boto +import tempfile +try: + import hashlib + _hashfn = hashlib.sha512 +except ImportError: + import md5 + _hashfn = md5.md5 + +METADATA_PREFIX = 'x-amz-meta-' +AMAZON_HEADER_PREFIX = 'x-amz-' + +# generates the aws canonical string for the given parameters +def canonical_string(method, path, headers, expires=None): + interesting_headers = {} + for key in headers: + lk = key.lower() + if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX): + interesting_headers[lk] = headers[key].strip() + + # these keys get empty strings if they don't exist + if not interesting_headers.has_key('content-type'): + interesting_headers['content-type'] = '' + if not interesting_headers.has_key('content-md5'): + interesting_headers['content-md5'] = '' + + # just in case someone used this. it's not necessary in this lib. + if interesting_headers.has_key('x-amz-date'): + interesting_headers['date'] = '' + + # if you're using expires for query string auth, then it trumps date + # (and x-amz-date) + if expires: + interesting_headers['date'] = str(expires) + + sorted_header_keys = interesting_headers.keys() + sorted_header_keys.sort() + + buf = "%s\n" % method + for key in sorted_header_keys: + if key.startswith(AMAZON_HEADER_PREFIX): + buf += "%s:%s\n" % (key, interesting_headers[key]) + else: + buf += "%s\n" % interesting_headers[key] + + # don't include anything after the first ? in the resource... + buf += "%s" % path.split('?')[0] + + # ...unless there is an acl or torrent parameter + if re.search("[&?]acl($|=|&)", path): + buf += "?acl" + elif re.search("[&?]logging($|=|&)", path): + buf += "?logging" + elif re.search("[&?]torrent($|=|&)", path): + buf += "?torrent" + elif re.search("[&?]location($|=|&)", path): + buf += "?location" + + return buf + +def merge_meta(headers, metadata): + final_headers = headers.copy() + for k in metadata.keys(): + final_headers[METADATA_PREFIX + k] = metadata[k] + + return final_headers + +def get_aws_metadata(headers): + metadata = {} + for hkey in headers.keys(): + if hkey.lower().startswith(METADATA_PREFIX): + metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey] + del headers[hkey] + return metadata + +def retry_url(url): + for i in range(0, 10): + try: + req = urllib2.Request(url) + resp = urllib2.urlopen(req) + return resp.read() + except: + boto.log.exception('Caught exception reading instance data') + time.sleep(2**i) + boto.log.error('Unable to read instance data, giving up') + return '' + +def get_instance_metadata(version='latest'): + metadata = {} + url = 'http://169.254.169.254/%s/meta-data/' % version + data = retry_url(url) + if data: + md_fields = data.split('\n') + for md in md_fields: + val = retry_url(url + md) + if val.find('\n') > 0: + val = val.split('\n') + metadata[md] = val + return metadata + +def get_instance_userdata(version='latest', sep=None): + url = 'http://169.254.169.254/%s/user-data/' % version + user_data = retry_url(url) + if user_data: + if sep: + l = user_data.split(sep) + user_data = {} + for nvpair in l: + t = nvpair.split('=') + user_data[t[0].strip()] = t[1].strip() + return user_data + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +def get_ts(ts=None): + if not ts: + ts = time.gmtime() + return time.strftime(ISO8601, ts) + +def parse_ts(ts): + return datetime.datetime.strptime(ts, ISO8601) + +def find_class(module_name, class_name=None): + if class_name: + module_name = "%s.%s" % (module_name, class_name) + modules = module_name.split('.') + path = None + c = None + + for m in modules[1:]: + if c: + c = getattr(c, m) + else: + c = getattr(__import__(".".join(modules[0:-1])), m) + return c + +def update_dme(username, password, dme_id, ip_address): + """ + Update your Dynamic DNS record with DNSMadeEasy.com + """ + dme_url = 'https://www.dnsmadeeasy.com/servlet/updateip' + dme_url += '?username=%s&password=%s&id=%s&ip=%s' + s = urllib2.urlopen(dme_url % (username, password, dme_id, ip_address)) + return s.read() + +def fetch_file(uri, file=None, username=None, password=None): + """ + Fetch a file based on the URI provided. If you do not pass in a file pointer + a tempfile.NamedTemporaryFile, or None if the file could not be + retrieved is returned. + The URI can be either an HTTP url, or "s3://bucket_name/key_name" + """ + boto.log.info('Fetching %s' % uri) + if file == None: + file = tempfile.NamedTemporaryFile() + try: + working_dir = boto.config.get("General", "working_dir") + if uri.startswith('s3://'): + bucket_name, key_name = uri[len('s3://'):].split('/') + c = boto.connect_s3() + bucket = c.get_bucket(bucket_name) + key = bucket.get_key(key_name) + key.get_contents_to_file(file) + else: + if username and password: + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + passman.add_password(None, uri, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + s = urllib2.urlopen(uri) + file.write(s.read()) + file.seek(0) + except: + raise + boto.log.exception('Problem Retrieving file: %s' % uri) + file = None + return file + +class ShellCommand(object): + + def __init__(self, command, wait=True): + self.exit_code = 0 + self.command = command + self.log_fp = StringIO.StringIO() + self.wait = wait + self.run() + + def run(self): + boto.log.info('running:%s' % self.command) + self.process = popen2.Popen4(self.command) + if(self.wait): + return self.getStatus() + + def setReadOnly(self, value): + raise AttributeError + + def getStatus(self): + if(not self.exit_code): + status = self.process.wait() + self.log_fp.write(self.process.fromchild.read()) + boto.log.info(self.log_fp.getvalue()) + self.exit_code = os.WEXITSTATUS(status) + return self.exit_code + + status = property(getStatus, setReadOnly, None, 'The exit code for the command') + + def getOutput(self): + return self.log_fp.getvalue() + + output = property(getOutput, setReadOnly, None, 'The STDIN and STDERR output of the command') + +class AuthSMTPHandler(logging.handlers.SMTPHandler): + """ + This class extends the SMTPHandler in the standard Python logging module + to accept a username and password on the constructor and to then use those + credentials to authenticate with the SMTP server. To use this, you could + add something like this in your boto config file: + + [handler_hand07] + class=boto.utils.AuthSMTPHandler + level=WARN + formatter=form07 + args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') + """ + + def __init__(self, mailhost, username, password, fromaddr, toaddrs, subject): + """ + Initialize the handler. + + We have extended the constructor to accept a username/password + for SMTP authentication. + """ + logging.handlers.SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject) + self.username = username + self.password = password + + def emit(self, record): + """ + Emit a record. + + Format the record and send it to the specified addressees. + It would be really nice if I could add authorization to this class + without having to resort to cut and paste inheritance but, no. + """ + try: + import smtplib + try: + from email.Utils import formatdate + except: + formatdate = self.date_time + port = self.mailport + if not port: + port = smtplib.SMTP_PORT + smtp = smtplib.SMTP(self.mailhost, port) + smtp.login(self.username, self.password) + msg = self.format(record) + msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( + self.fromaddr, + string.join(self.toaddrs, ","), + self.getSubject(record), + formatdate(), msg) + smtp.sendmail(self.fromaddr, self.toaddrs, msg) + smtp.quit() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + +class LRUCache(dict): + """A dictionary-like object that stores only a certain number of items, and + discards its least recently used item when full. + + >>> cache = LRUCache(3) + >>> cache['A'] = 0 + >>> cache['B'] = 1 + >>> cache['C'] = 2 + >>> len(cache) + 3 + + >>> cache['A'] + 0 + + Adding new items to the cache does not increase its size. Instead, the least + recently used item is dropped: + + >>> cache['D'] = 3 + >>> len(cache) + 3 + >>> 'B' in cache + False + + Iterating over the cache returns the keys, starting with the most recently + used: + + >>> for key in cache: + ... print key + D + A + C + + This code is based on the LRUCache class from Genshi which is based on + Mighty's LRUCache from ``myghtyutils.util``, written + by Mike Bayer and released under the MIT license (Genshi uses the + BSD License). See: + + http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py + """ + + class _Item(object): + def __init__(self, key, value): + self.previous = self.next = None + self.key = key + self.value = value + def __repr__(self): + return repr(self.value) + + def __init__(self, capacity): + self._dict = dict() + self.capacity = capacity + self.head = None + self.tail = None + + def __contains__(self, key): + return key in self._dict + + def __iter__(self): + cur = self.head + while cur: + yield cur.key + cur = cur.next + + def __len__(self): + return len(self._dict) + + def __getitem__(self, key): + item = self._dict[key] + self._update_item(item) + return item.value + + def __setitem__(self, key, value): + item = self._dict.get(key) + if item is None: + item = self._Item(key, value) + self._dict[key] = item + self._insert_item(item) + else: + item.value = value + self._update_item(item) + self._manage_size() + + def __repr__(self): + return repr(self._dict) + + def _insert_item(self, item): + item.previous = None + item.next = self.head + if self.head is not None: + self.head.previous = item + else: + self.tail = item + self.head = item + self._manage_size() + + def _manage_size(self): + while len(self._dict) > self.capacity: + olditem = self._dict[self.tail.key] + del self._dict[self.tail.key] + if self.tail != self.head: + self.tail = self.tail.previous + self.tail.next = None + else: + self.head = self.tail = None + + def _update_item(self, item): + if self.head == item: + return + + previous = item.previous + previous.next = item.next + if item.next is not None: + item.next.previous = previous + else: + self.tail = previous + + item.previous = None + item.next = self.head + self.head.previous = self.head = item + +class Password: + """ + Password object that stores itself as SHA512 hashed. + """ + def __init__(self, str=None): + """ + Load the string from an initial value, this should be the raw SHA512 hashed password + """ + self.str = str + + def set(self, value): + self.str = _hashfn(value).hexdigest() + + def __str__(self): + return str(self.str) + + def __eq__(self, other): + return str(_hashfn(other).hexdigest()) == str(self.str) + + def __len__(self): + return len(self.str) diff --git a/thirdparty/duplicity/Homepage.txt b/thirdparty/duplicity/Homepage.txt new file mode 100644 index 000000000..5f4fd5b96 --- /dev/null +++ b/thirdparty/duplicity/Homepage.txt @@ -0,0 +1 @@ +http://www.nongnu.org/duplicity/duplicity.1.html \ No newline at end of file diff --git a/thirdparty/duplicity/License.txt b/thirdparty/duplicity/License.txt new file mode 100644 index 000000000..94a9ed024 --- /dev/null +++ b/thirdparty/duplicity/License.txt @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/thirdparty/duplicity/duplicity.py b/thirdparty/duplicity/duplicity.py new file mode 100644 index 000000000..4f0a0b1f8 --- /dev/null +++ b/thirdparty/duplicity/duplicity.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python +# duplicity -- Encrypted bandwidth efficient backup +# Version 0.4.10 released September 29, 2002 +# +# Copyright (C) 2002 Ben Escoto +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# See http://www.nongnu.org/duplicity for more information. +# Please send mail to me or the mailing list if you find bugs or have +# any suggestions. + +from __future__ import generators +import getpass, gzip, os, sys, time, types +from duplicity import collections, commandline, diffdir, dup_temp, \ + dup_time, file_naming, globals, gpg, log, manifest, patchdir, \ + path, robust, tempdir + +# If exit_val is not None, exit with given value at end. +exit_val = None + + +def get_passphrase(): + """Get passphrase from environment or, failing that, from user""" + try: + return os.environ['PASSPHRASE'] + except KeyError: + pass + + log.Log("PASSPHRASE variable not set, asking user.", 5) + while 1: + pass1 = getpass.getpass("GnuPG passphrase: ") + pass2 = getpass.getpass("Retype to confirm: ") + if not pass1 == pass2: + print "First and second passphrases do not match! Please try again." + continue + if not pass1 and not globals.gpg_profile.recipients: + print "Cannot use empty passphrase with symmetric encryption! Please try again." + continue + return pass1 + + +def write_multivol(backup_type, tarblock_iter, backend): + """Encrypt volumes of tarblock_iter and write to backend + + backup_type should be "inc" or "full" and only matters here when + picking the filenames. The path_prefix will determine the names + of the files written to backend. Also writes manifest file. + Returns number of bytes written. + + """ + def get_indicies(tarblock_iter): + """Return start_index and end_index of previous volume""" + start_index = tarblock_iter.recall_index() + if start_index is None: + start_index = () + end_index = tarblock_iter.get_previous_index() + if end_index is None: + end_index = start_index + return start_index, end_index + + mf = manifest.Manifest().set_dirinfo() + vol_num = 1; bytes_written = 0; at_end = 0 + while not at_end: + # Create volume + tarblock_iter.remember_next_index() # keep track of start index + dest_filename = file_naming.get(backup_type, vol_num, + encrypted = globals.encryption, + gzipped = not globals.encryption) + tdp = dup_temp.new_tempduppath(file_naming.parse(dest_filename)) + if globals.encryption: + at_end = gpg.GPGWriteFile(tarblock_iter, tdp.name, + globals.gpg_profile,globals.volsize) + else: + at_end = gpg.GzipWriteFile(tarblock_iter, tdp.name,globals.volsize) + tdp.setdata() + + # Add volume information to manifest + vi = manifest.VolumeInfo() + start_index, end_index = get_indicies(tarblock_iter) + vi.set_info(vol_num, start_index, end_index) + vi.set_hash("SHA1", gpg.get_hash("SHA1", tdp)) + mf.add_volume_info(vi) + + backend.put(tdp, dest_filename) + vol_num += 1; bytes_written += tdp.getsize() + tdp.delete() + + bytes_written += write_manifest(mf, backup_type, backend) + return bytes_written + + +def write_manifest(mf, backup_type, backend): + """Write manifest to file in archive_dir and encrypted to backend + + Returns number of bytes written + + """ + mf_string = mf.to_string() + if globals.archive_dir: + local_mf_name = file_naming.get(backup_type, manifest = 1) + fin = dup_temp.get_fileobj_duppath(globals.archive_dir, local_mf_name) + fin.write(mf_string) + fin.close() + + sizelist = [] # will hold length of file in bytes + remote_mf_name = file_naming.get(backup_type, manifest = 1, + encrypted = globals.encryption) + remote_fin = backend.get_fileobj_write(remote_mf_name, sizelist = sizelist) + remote_fin.write(mf_string) + remote_fin.close() + return sizelist[0] + + +def get_sig_fileobj(sig_type): + """Return a fileobj opened for writing, save results as signature + + If globals.archive_dir is available, save signatures there + gzipped. Save them on the backend encrypted as needed. + + """ + assert sig_type == "full-sig" or sig_type == "new-sig" + + sig_filename = file_naming.get(sig_type, encrypted = globals.encryption, + gzipped = not globals.encryption) + fh = globals.backend.get_fileobj_write(sig_filename) + + # by MDR. This was changed to use addfilehandle so we get both + # remote and local sig files + if globals.archive_dir: + local_sig_filename = file_naming.get(sig_type, gzipped = 1) + fh.addfilehandle(dup_temp.get_fileobj_duppath(globals.archive_dir, + local_sig_filename)) + return fh + + +def full_backup(col_stats): + """Do full backup of directory to backend, using archive_dir""" + sig_outfp = get_sig_fileobj("full-sig") + tarblock_iter = diffdir.DirFull_WriteSig(globals.select, sig_outfp) + bytes_written = write_multivol("full", tarblock_iter, globals.backend) + sig_outfp.close() + col_stats.set_values(sig_chain_warning = None).cleanup_signatures() + print_statistics(diffdir.stats, bytes_written) + + +def check_sig_chain(col_stats): + """Get last signature chain for inc backup, or None if none available""" + if not col_stats.matched_chain_pair: + if globals.incremental: + log.FatalError("Fatal Error: Unable to start incremental backup. " + "Old signatures not found and incremental specified") + else: + log.Warn("No signatures found, switching to full backup.") + return None + return col_stats.matched_chain_pair[0] + + +def print_statistics(stats, bytes_written): + """If globals.print_statistics, print stats after adding bytes_written""" + if globals.print_statistics: + diffdir.stats.TotalDestinationSizeChange = bytes_written + print diffdir.stats.get_stats_logstring("Backup Statistics") + + +def incremental_backup(sig_chain): + """Do incremental backup of directory to backend, using archive_dir""" + dup_time.setprevtime(sig_chain.end_time) + new_sig_outfp = get_sig_fileobj("new-sig") + tarblock_iter = diffdir.DirDelta_WriteSig(globals.select, + sig_chain.get_fileobjs(), new_sig_outfp) + bytes_written = write_multivol("inc", tarblock_iter, globals.backend) + new_sig_outfp.close() + print_statistics(diffdir.stats, bytes_written) + + +def list_current(col_stats): + """List the files current in the archive (examining signature only)""" + sig_chain = check_sig_chain(col_stats) + if not sig_chain: + log.FatalError("No signature data found, unable to list files.") + path_iter = diffdir.get_combined_path_iter(sig_chain.get_fileobjs()) + for path in path_iter: + if path.difftype != "deleted": + print dup_time.timetopretty(path.getmtime()), \ + path.get_relative_path() + + +def restore(col_stats): + """Restore archive in globals.backend to globals.local_path""" + if not patchdir.Write_ROPaths(globals.local_path, + restore_get_patched_rop_iter(col_stats)): + if globals.restore_dir: + log.FatalError("%s not found in archive, no files restored." + % (globals.restore_dir,)) + else: + log.FatalError("No files found in archive - nothing restored.") + + +def restore_get_patched_rop_iter(col_stats): + """Return iterator of patched ROPaths of desired restore data""" + if globals.restore_dir: + index = tuple(globals.restore_dir.split("/")) + else: + index = () + time = globals.restore_time or dup_time.curtime + backup_chain = col_stats.get_backup_chain_at_time(time) + assert backup_chain, col_stats.all_backup_chains + backup_setlist = backup_chain.get_sets_at_time(time) + + def get_fileobj_iter(backup_set): + """Get file object iterator from backup_set contain given index""" + manifest = backup_set.get_manifest() + for vol_num in manifest.get_containing_volumes(index): + yield restore_get_enc_fileobj(backup_set.backend, + backup_set.volume_name_dict[vol_num], + manifest.volume_info_dict[vol_num]) + + fileobj_iters = map(get_fileobj_iter, backup_setlist) + tarfiles = map(patchdir.TarFile_FromFileobjs, fileobj_iters) + return patchdir.tarfiles2rop_iter(tarfiles, index) + + +def restore_get_enc_fileobj(backend, filename, volume_info): + """Return plaintext fileobj from encrypted filename on backend + + If volume_info is set, the hash of the file will be checked, + assuming some hash is available. Also, if globals.sign_key is + set, a fatal error will be raised if file not signed by sign_key. + + """ + parseresults = file_naming.parse(filename) + tdp = dup_temp.new_tempduppath(parseresults) + backend.get(filename, tdp) + restore_check_hash(volume_info, tdp) + + fileobj = tdp.filtered_open_with_delete("rb") + if parseresults.encrypted and globals.gpg_profile.sign_key: + restore_add_sig_check(fileobj) + return fileobj + + +def restore_check_hash(volume_info, vol_path): + """Check the hash of vol_path path against data in volume_info""" + hash_pair = volume_info.get_best_hash() + if hash_pair: + calculated_hash = gpg.get_hash(hash_pair[0], vol_path) + if calculated_hash != hash_pair[1]: + log.FatalError("Invalid data - %s hash mismatch:\n" + "Calculated hash: %s\n" + "Manifest hash: %s\n" % + (hash_pair[0], calculated_hash, hash_pair[1])) + + +def restore_add_sig_check(fileobj): + """Require signature when closing fileobj matches sig in gpg_profile""" + assert (isinstance(fileobj, dup_temp.FileobjHooked) and + isinstance(fileobj.fileobj, gpg.GPGFile)), fileobj + def check_signature(): + """Thunk run when closing volume file""" + actual_sig = fileobj.fileobj.get_signature() + if actual_sig != globals.gpg_profile.sign_key: + log.FatalError("Volume was not signed by key %s, not %s" % + (actual_sig, globals.gpg_profile.sign_key)) + fileobj.addhook(check_signature) + + +def verify(col_stats): + """Verify files, logging differences""" + global exit_val + collated = diffdir.collate2iters(restore_get_patched_rop_iter(col_stats), + globals.select) + diff_count = 0; total_count = 0 + for backup_ropath, current_path in collated: + if not backup_ropath: + backup_ropath = path.ROPath(current_path.index) + if not current_path: + current_path = path.ROPath(backup_ropath.index) + if not backup_ropath.compare_verbose(current_path): + diff_count += 1 + total_count += 1 + log.Log("Verify complete: %s %s compared, %s %s found." % + (total_count, total_count == 1 and "file" or "files", + diff_count, diff_count == 1 and "difference" or "differences"), 3) + if diff_count >= 1: + exit_val = 1 + + +def cleanup(col_stats): + """Delete the extraneous files in the current backend""" + extraneous = col_stats.get_extraneous() + if not extraneous: + log.Warn("No extraneous files found, nothing deleted in cleanup.") + return + + filestr = "\n".join(extraneous) + if globals.force: + if len(extraneous) > 1: + log.Log("Deleting these files from backend:\n"+filestr, 3) + else: + log.Log("Deleting this file from backend:\n"+filestr, 3) + col_stats.backend.delete(extraneous) + else: + if len(extraneous) > 1: + log.Warn("Found the following files to delete:") + else: + log.Warn("Found the following file to delete:") + log.Warn(filestr + "\nRun duplicity again with the --force " + "option to actually delete.") + +def remove_all_but_n_full(col_stats): + "Remove backup files older than the last n full backups." + assert globals.keep_chains is not None + + globals.remove_time = col_stats.get_nth_last_full_backup_time(globals.keep_chains) + + return remove_old(col_stats) + +def remove_old(col_stats): + """Remove backup files older than globals.remove_time from backend""" + assert globals.remove_time is not None + def set_times_str(setlist): + """Return string listing times of sets in setlist""" + return "\n".join(map(lambda s: dup_time.timetopretty(s.get_time()), + setlist)) + + req_list = col_stats.get_older_than_required(globals.remove_time) + if req_list: + log.Warn("There are backup set(s) at time(s):\n%s\nWhich can't be " + "deleted because newer sets depend on them." % + set_times_str(req_list)) + + if (col_stats.matched_chain_pair and + col_stats.matched_chain_pair[1].end_time < globals.remove_time): + log.Warn("Current active backup chain is older than specified time.\n" + "However, it will not be deleted. To remove all your backups,\n" + "manually purge the repository.") + + setlist = col_stats.get_older_than(globals.remove_time) + if not setlist: + log.Warn("No old backup sets found, nothing deleted.") + return + if globals.force: + if len(setlist) > 1: + log.Log("Deleting backup sets at times:\n" + + set_times_str(setlist), 3) + else: + log.Log("Deleting backup set at times:\n" + + set_times_str(setlist), 3) + setlist.reverse() # save oldest for last + for set in setlist: + set.delete() + col_stats.set_values(sig_chain_warning = None).cleanup_signatures() + else: + if len(setlist) > 1: + log.Warn("Found old backup sets at the following times:") + else: + log.Warn("Found old backup set at the following time:") + log.Warn(set_times_str(setlist) + + "\nRerun command with --force option to actually delete.") + + +def check_last_manifest(col_stats): + """Check consistency and hostname/directory of last manifest""" + if not col_stats.all_backup_chains: + return + last_backup_set = col_stats.all_backup_chains[-1].get_last() + last_backup_set.check_manifests() + + +def main(): + """Start/end here""" + dup_time.setcurtime() + action = commandline.ProcessCommandLine(sys.argv[1:]) + col_stats = collections.CollectionsStatus(globals.backend, + globals.archive_dir).set_values() + log.Log("Collection Status\n-----------------\n" + str(col_stats), 8) + + last_full_time = col_stats.get_last_full_backup_time() + if last_full_time > 0: + log.Log("Last full backup date: " + dup_time.timetopretty(last_full_time), 4) + else: + log.Log("Last full backup date: none", 4) + if action == "inc" and last_full_time < globals.full_force_time: + log.Log("Last full backup is too old, forcing full backup", 3) + action = "full" + + os.umask(077) + + # for public key encryption (without signing!), no passphrase is required. + pubkey_only = (not globals.gpg_profile.sign_key and + globals.gpg_profile.recipients and + globals.encryption) + + # cases where we do not need to get a passphrase: + # full: with pubkey enc. doesn't depend on old encrypted info + # inc and pubkey enc.: need a manifest, which the archive dir has unencrypted + # with encryption disabled + # listing files: needs a manifest, but the archive dir has that + # collection status only looks at a repository + if ((action == "full" and pubkey_only) or + (action == "inc" and pubkey_only and globals.archive_dir) or + (not globals.encryption) or + (action == "list-current" and globals.archive_dir) or + action in ["collection-status", + "remove-older-then", + "remove-all-but-n-full", + ]): + globals.gpg_profile.passphrase = "" + else: + globals.gpg_profile.passphrase = get_passphrase() + + if action == "restore": + restore(col_stats) + elif action == "verify": + verify(col_stats) + elif action == "list-current": + list_current(col_stats) + elif action == "collection-status": + print str(col_stats) + elif action == "cleanup": + cleanup(col_stats) + elif action == "remove-old": + remove_old(col_stats) + elif action == "remove-all-but-n-full": + remove_all_but_n_full(col_stats) + else: + assert action == "inc" or action == "full", action + if action == "full": + full_backup(col_stats) + else: + check_last_manifest(col_stats) # not needed for full backup + sig_chain = check_sig_chain(col_stats) + if not sig_chain: + full_backup(col_stats) + else: + incremental_backup(sig_chain) + globals.backend.close() + if exit_val is not None: + sys.exit(exit_val) + +def with_tempdir(fn): + globals.badfiles = [] + try: + fn() + finally: + for i in globals.badfiles: + try: + os.unlink(i) + except: + pass + + tempdir.default().cleanup() + +if __name__ == "__main__": + with_tempdir(main) diff --git a/thirdparty/duplicity/duplicity/__init__.py b/thirdparty/duplicity/duplicity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/thirdparty/duplicity/duplicity/_librsync.pyd b/thirdparty/duplicity/duplicity/_librsync.pyd new file mode 100644 index 000000000..605c6d53f Binary files /dev/null and b/thirdparty/duplicity/duplicity/_librsync.pyd differ diff --git a/thirdparty/duplicity/duplicity/backends.py b/thirdparty/duplicity/duplicity/backends.py new file mode 100644 index 000000000..8be95d491 --- /dev/null +++ b/thirdparty/duplicity/duplicity/backends.py @@ -0,0 +1,1039 @@ +# Copyright 2002 Ben Escoto +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +"""Provides functions and classes for getting/sending files to destination""" + +import os, socket, types, tempfile, time, sys +import log, path, dup_temp, file_naming, atexit +import base64, getpass, xml.dom.minidom, httplib, urllib +import socket, globals, re, string +from duplicity import tempdir + +import urlparse_2_5 as urlparser + +socket.setdefaulttimeout(globals.timeout) + +class BackendException(Exception): pass +class ParsingException(Exception): pass + + +def straight_url(parsed_url): + """Return a URL from a urlparse object without a username or password.""" + + # Get a copy of the network location without the username or password. + straight_netloc = parsed_url.netloc.split('@')[-1] + + # Replace the full network location with the stripped copy. + return parsed_url.geturl().replace(parsed_url.netloc, straight_netloc, 1) + + +def ParsedUrl(url_string): + # These URL schemes have a backend with a notion of an RFC "network location". + # The 'file' and 's3+http' schemes should not be in this list. + # 'http' and 'https' are not actually used for duplicity backend urls, but are needed + # in order to properly support urls returned from some webdav servers. adding them here + # is a hack. we should instead not stomp on the url parsing module to begin with. + urlparser.uses_netloc = [ 'ftp', 'hsi', 'rsync', 's3', 'scp', 'ssh', 'webdav', 'webdavs', 'http', 'https' ] + + # Do not transform or otherwise parse the URL path component. + urlparser.uses_query = [] + urlparser.uses_fragment = [] + + pu = urlparser.urlparse(url_string) + + # This happens for implicit local paths. + if not pu.scheme: + return pu + + # Our backends do not handle implicit hosts. + if pu.scheme in urlparser.uses_netloc and not pu.hostname: + log.FatalError('Bad %s:// URL syntax: %s' % (pu.scheme, url_string)) + + # Our backends do not handle implicit relative paths. + if not pu.scheme in urlparser.uses_netloc and not pu.path.startswith('//'): + log.FatalError('Bad %s:// URL syntax: %s' % (pu.scheme, url_string)) + + return pu + + +def get_backend(url_string): + """Return Backend object from url string, or None if not a url string""" + """If a protocol is unsupported a fatal error will be raised.""" + + pu = ParsedUrl(url_string) + + # This happens for implicit local paths. + if not pu.scheme: + return None + + global protocol_class_dict + try: + backend_class = protocol_class_dict[pu.scheme] + except KeyError: + log.FatalError("Unknown scheme '%s'" % (pu.scheme,)) + return backend_class(pu) + + +class Backend: + """Represent a connection to the destination device/computer + + Classes that subclass this should implement the put, get, list, + and delete methods. + + """ + def __init__(self, parsed_url): + self.parsed_url = parsed_url + + def put(self, source_path, remote_filename = None): + """Transfer source_path (Path object) to remote_filename (string) + + If remote_filename is None, get the filename from the last + path component of pathname. + + """ + if not remote_filename: remote_filename = source_path.get_filename() + pass + + def get(self, remote_filename, local_path): + """Retrieve remote_filename and place in local_path""" + local_path.setdata() + pass + + def list(self): + """Return list of filenames (strings) present in backend""" + pass + + def delete(self, filename_list): + """Delete each filename in filename_list, in order if possible""" + pass + + def get_password(self): + """Get a password from the target url or from the environment.""" + + if self.parsed_url.password: + return self.parsed_url.password + + try: + password = os.environ['FTP_PASSWORD'] + except KeyError: + password = getpass.getpass("Password for '%s': " % self.parsed_url.hostname) + os.environ['FTP_PASSWORD'] = password + return password + + def munge_password(self, commandline): + """Remove password from commandline""" + if self.parsed_url.password: + return re.sub(self.parsed_url.password, '', commandline) + else: + return commandline + + def run_command(self, commandline): + """Run given commandline with logging and error detection""" + private = self.munge_password(commandline) + log.Log("Running '%s'" % private, 5) + if os.system(commandline): + raise BackendException("Error running '%s'" % private) + + def run_command_persist(self, commandline): + """Run given commandline with logging and error detection + repeating it several times if it fails""" + private = self.munge_password(commandline) + for n in range(1, globals.num_retries+1): + log.Log("Running '%s' (attempt #%d)" % (private, n), 5) + if not os.system(commandline): + return + log.Log("Running '%s' failed (attempt #%d)" % (private, n), 1) + time.sleep(30) + log.Log("Giving up trying to execute '%s' after %d attempts" % (private, globals.num_retries), 1) + raise BackendException("Error running '%s'" % private) + + def popen(self, commandline): + """Run command and return stdout results""" + private = self.munge_password(commandline) + log.Log("Reading results of '%s'" % private, 5) + fout = os.popen(commandline) + results = fout.read() + if fout.close(): + raise BackendException("Error running '%s'" % private) + return results + + def popen_persist(self, commandline): + """Run command and return stdout results, repeating on failure""" + private = self.munge_password(commandline) + for n in range(1, globals.num_retries+1): + log.Log("Reading results of '%s'" % private, 5) + fout = os.popen(commandline) + results = fout.read() + result_status = fout.close() + if not result_status: + return results + elif result_status == 1280 and self.parsed_url.scheme == 'ftp': + # This squelches the "file not found" result fromm ncftpls when + # the ftp backend looks for a collection that does not exist. + return '' + log.Log("Running '%s' failed (attempt #%d)" % (private, n), 1) + time.sleep(30) + log.Log("Giving up trying to execute '%s' after %d attempts" % (private, globals.num_retries), 1) + raise BackendException("Error running '%s'" % private) + + def get_fileobj_read(self, filename, parseresults = None): + """Return fileobject opened for reading of filename on backend + + The file will be downloaded first into a temp file. When the + returned fileobj is closed, the temp file will be deleted. + + """ + if not parseresults: + parseresults = file_naming.parse(filename) + assert parseresults, "Filename not correctly parsed" + tdp = dup_temp.new_tempduppath(parseresults) + self.get(filename, tdp) + tdp.setdata() + return tdp.filtered_open_with_delete("rb") + + def get_fileobj_write(self, filename, parseresults = None, + sizelist = None): + """Return fileobj opened for writing, write to backend on close + + The file will be encoded as specified in parseresults (or as + read from the filename), and stored in a temp file until it + can be copied over and deleted. + + If sizelist is not None, it should be set to an empty list. + The number of bytes will be inserted into the list. + + """ + if not parseresults: + parseresults = file_naming.parse(filename) + assert parseresults, "Filename %s not correctly parsed" % filename + tdp = dup_temp.new_tempduppath(parseresults) + + def close_file_hook(): + """This is called when returned fileobj is closed""" + self.put(tdp, filename) + if sizelist is not None: + tdp.setdata() + sizelist.append(tdp.getsize()) + tdp.delete() + + fh = dup_temp.FileobjHooked(tdp.filtered_open("wb")) + fh.addhook(close_file_hook) + return fh + + def get_data(self, filename, parseresults = None): + """Retrieve a file from backend, process it, return contents""" + fin = self.get_fileobj_read(filename, parseresults) + buf = fin.read() + assert not fin.close() + return buf + + def put_data(self, buffer, filename, parseresults = None): + """Put buffer into filename on backend after processing""" + fout = self.get_fileobj_write(filename, parseresults) + fout.write(buffer) + assert not fout.close() + + def close(self): + """This is called when a connection is no longer needed""" + pass + + +class LocalBackend(Backend): + """Use this backend when saving to local disk + + Urls look like file://testfiles/output. Relative to root can be + gotten with extra slash (file:///usr/local). + + """ + def __init__(self, parsed_url): + Backend.__init__(self, parsed_url) + # The URL form "file:MyFile" is not a valid duplicity target. + if not parsed_url.path.startswith( '//' ): + raise BackendException( "Bad file:// path syntax." ) + self.remote_pathdir = path.Path(parsed_url.path[2:]) + + def put(self, source_path, remote_filename = None, rename = None): + """If rename is set, try that first, copying if doesn't work""" + if not remote_filename: remote_filename = source_path.get_filename() + target_path = self.remote_pathdir.append(remote_filename) + log.Log("Writing %s" % target_path.name, 6) + if rename: + try: source_path.rename(target_path) + except OSError: pass + else: return + target_path.writefileobj(source_path.open("rb")) + + def get(self, filename, local_path): + """Get file and put in local_path (Path object)""" + source_path = self.remote_pathdir.append(filename) + local_path.writefileobj(source_path.open("rb")) + + def list(self): + """List files in that directory""" + try: + os.makedirs(self.remote_pathdir.base) + except: + pass + return self.remote_pathdir.listdir() + + def delete(self, filename_list): + """Delete all files in filename list""" + assert type(filename_list) is not types.StringType + try: + for filename in filename_list: + self.remote_pathdir.append(filename).delete() + except OSError, e: raise BackendException(str(e)) + + +# The following can be redefined to use different shell commands from +# ssh or scp or to add more arguments. However, the replacements must +# have the same syntax. Also these strings will be executed by the +# shell, so shouldn't have strange characters in them. +scp_command = "scp" +sftp_command = "sftp" + +# default to batch mode using public-key encryption +ssh_askpass = False + +# user added ssh options +ssh_options = "" + +class sshBackend(Backend): + """This backend copies files using scp. List not supported""" + def __init__(self, parsed_url): + """scpBackend initializer""" + Backend.__init__(self, parsed_url) + try: + import pexpect + self.pexpect = pexpect + except ImportError: + self.pexpect = None + if not (self.pexpect and + hasattr(self.pexpect, '__version__') and + self.pexpect.__version__ >= '2.1'): + log.FatalError("This backend requires the pexpect module version 2.1 or later." + "You can get pexpect from http://pexpect.sourceforge.net or " + "python-pexpect from your distro's repository.") + + # host string of form user@hostname:port + self.host_string = parsed_url.netloc + # make sure remote_dir is always valid + if parsed_url.path: + # remove leading '/' + self.remote_dir = re.sub(r'^/', r'', parsed_url.path, 1) + else: + self.remote_dir = '.' + self.remote_prefix = self.remote_dir + '/' + # maybe use different ssh port + if parsed_url.port: + self.ssh_options = ssh_options + " -oPort=%s" % parsed_url.port + else: + self.ssh_options = ssh_options + # set up password + if ssh_askpass: + self.password = self.get_password() + else: + self.password = '' + + def run_scp_command(self, commandline): + """ Run an scp command, responding to password prompts """ + for n in range(1, globals.num_retries+1): + log.Log("Running '%s' (attempt #%d)" % (commandline, n), 5) + child = self.pexpect.spawn(commandline, timeout = globals.timeout) + cmdloc = 0 + if ssh_askpass: + state = "authorizing" + else: + state = "copying" + while 1: + if state == "authorizing": + match = child.expect([self.pexpect.EOF, + self.pexpect.TIMEOUT, + "(?i)password:", + "(?i)permission denied", + "authenticity"], + timeout = globals.timeout) + log.Log("State = %s, Before = '%s'" % (state, child.before.strip()), 9) + if match == 0: + log.Log("Failed to authenticate", 5) + break + elif match == 1: + log.Log("Timeout waiting to authenticate", 5) + break + elif match == 2: + child.sendline(self.password) + state = "copying" + elif match == 3: + log.Log("Invalid SSH password", 1) + break + elif match == 4: + log.Log("Remote host authentication failed (missing known_hosts entry?)", 1) + break + elif state == "copying": + match = child.expect([self.pexpect.EOF, + self.pexpect.TIMEOUT, + "stalled", + "authenticity", + "ETA"], + timeout = globals.timeout) + log.Log("State = %s, Before = '%s'" % (state, child.before.strip()), 9) + if match == 0: + break + elif match == 1: + log.Log("Timeout waiting for response", 5) + break + elif match == 2: + state = "stalled" + elif match == 3: + log.Log("Remote host authentication failed (missing known_hosts entry?)", 1) + break + elif state == "stalled": + match = child.expect([self.pexpect.EOF, + self.pexpect.TIMEOUT, + "ETA"], + timeout = globals.timeout) + log.Log("State = %s, Before = '%s'" % (state, child.before.strip()), 9) + if match == 0: + break + elif match == 1: + log.Log("Stalled for too long, aborted copy", 5) + break + elif match == 2: + state = "copying" + child.close(force = True) + if child.exitstatus == 0: + return + log.Log("Running '%s' failed (attempt #%d)" % (commandline, n), 1) + time.sleep(30) + log.Log("Giving up trying to execute '%s' after %d attempts" % (commandline, globals.num_retries), 1) + raise BackendException("Error running '%s'" % commandline) + + def run_sftp_command(self, commandline, commands): + """ Run an sftp command, responding to password prompts, passing commands from list """ + for n in range(1, globals.num_retries+1): + log.Log("Running '%s' (attempt #%d)" % (commandline, n), 5) + child = self.pexpect.spawn(commandline, timeout = globals.timeout) + cmdloc = 0 + while 1: + match = child.expect([self.pexpect.EOF, + self.pexpect.TIMEOUT, + "sftp>", + "(?i)password:", + "(?i)permission denied", + "authenticity", + "(?i)no such file or directory"]) + log.Log("State = sftp, Before = '%s'" % (child.before.strip()), 9) + if match == 0: + break + elif match == 1: + log.Log("Timeout waiting for response", 5) + break + if match == 2: + if cmdloc < len(commands): + command = commands[cmdloc] + log.Log("sftp command: '%s'" % (command,), 5) + child.sendline(command) + cmdloc += 1 + else: + command = 'quit' + child.sendline(command) + res = child.before + elif match == 3: + child.sendline(self.password) + elif match == 4: + log.Log("Invalid SSH password", 1) + break + elif match == 5: + log.Log("Host key authenticity could not be verified (missing known_hosts entry?)", 1) + break + elif match == 6: + log.Log("Remote file or directory '%s' does not exist" % self.remote_dir, 1) + break + child.close(force = True) + if child.exitstatus == 0: + return res + log.Log("Running '%s' failed (attempt #%d)" % (commandline, n), 1) + time.sleep(30) + log.Log("Giving up trying to execute '%s' after %d attempts" % (commandline, globals.num_retries), 1) + raise BackendException("Error running '%s'" % commandline) + + def put(self, source_path, remote_filename = None): + """Use scp to copy source_dir/filename to remote computer""" + if not remote_filename: remote_filename = source_path.get_filename() + commandline = "%s %s %s %s:%s%s" % \ + (scp_command, self.ssh_options, source_path.name, self.host_string, + self.remote_prefix, remote_filename) + self.run_scp_command(commandline) + + def get(self, remote_filename, local_path): + """Use scp to get a remote file""" + commandline = "%s %s %s:%s%s %s" % \ + (scp_command, self.ssh_options, self.host_string, self.remote_prefix, + remote_filename, local_path.name) + self.run_scp_command(commandline) + local_path.setdata() + if not local_path.exists(): + raise BackendException("File %s not found" % local_path.name) + + def list(self): + """List files available for scp + + Note that this command can get confused when dealing with + files with newlines in them, as the embedded newlines cannot + be distinguished from the file boundaries. + """ + commands = ["mkdir %s" % (self.remote_dir,), + "cd %s" % (self.remote_dir,), + "ls -1"] + commandline = ("%s %s %s" % (sftp_command, self.ssh_options, self.host_string)) + l = self.run_sftp_command(commandline, commands).split('\n')[1:] + return filter(lambda x: x, map(string.strip, l)) + + def delete(self, filename_list): + """Runs sftp rm to delete files. Files must not require quoting""" + commands = ["cd %s" % (self.remote_dir,)] + for fn in filename_list: + commands.append("rm %s" % fn) + commandline = ("%s %s %s" % (sftp_command, self.ssh_options, self.host_string)) + self.run_sftp_command(commandline, commands) + + +class ftpBackend(Backend): + """Connect to remote store using File Transfer Protocol""" + def __init__(self, parsed_url): + Backend.__init__(self, parsed_url) + + # we expect an error return, so go low-level and ignore it + try: + p = os.popen("ncftpls -v") + fout = p.read() + ret = p.close() + except: + pass + # the expected error is 8 in the high-byte and some output + if ret != 0x0800 or not fout: + log.FatalError("NcFTP not found: Please install NcFTP version 3.1.9 or later") + + # version is the second word of the first line + version = fout.split('\n')[0].split()[1] + if version < "3.1.9": + log.FatalError("NcFTP too old: Duplicity requires NcFTP version 3.1.9 or later") + log.Log("NcFTP version is %s" % version, 4) + + self.parsed_url = parsed_url + + self.url_string = straight_url(self.parsed_url) + + # Use an explicit directory name. + if self.url_string[-1] != '/': + self.url_string += '/' + + self.password = self.get_password() + + if globals.ftp_connection == 'regular': + self.conn_opt = '-E' + else: + self.conn_opt = '-F' + + self.tempfile, self.tempname = tempdir.default().mkstemp() + os.write(self.tempfile, "host %s\n" % self.parsed_url.hostname) + os.write(self.tempfile, "user %s\n" % self.parsed_url.username) + os.write(self.tempfile, "pass %s\n" % self.password) + os.close(self.tempfile) + self.flags = "-f %s %s -t %s" % \ + (self.tempname, self.conn_opt, globals.timeout) + if parsed_url.port != None and parsed_url.port != 21: + self.flags += " -P '%s'" % (parsed_url.port) + + def put(self, source_path, remote_filename = None): + """Transfer source_path to remote_filename""" + remote_path = os.path.join(urllib.unquote(self.parsed_url.path.lstrip('/')), remote_filename).rstrip() + commandline = "ncftpput %s -m -V -C '%s' '%s'" % \ + (self.flags, source_path.name, remote_path) + self.run_command_persist(commandline) + + def get(self, remote_filename, local_path): + """Get remote filename, saving it to local_path""" + remote_path = os.path.join(urllib.unquote(self.parsed_url.path), remote_filename).rstrip() + commandline = "ncftpget %s -V -C '%s' '%s' '%s'" % \ + (self.flags, self.parsed_url.hostname, remote_path.lstrip('/'), local_path.name) + self.run_command_persist(commandline) + local_path.setdata() + + def list(self): + """List files in directory""" + # try for a long listing to avoid connection reset + commandline = "ncftpls %s -l '%s'" % \ + (self.flags, self.url_string) + l = self.popen_persist(commandline).split('\n') + l = filter(lambda x: x, l) + if not l: + return l + # if long list is not empty, get short list of names only + commandline = "ncftpls -x '' %s '%s'" % \ + (self.flags, self.url_string) + l = self.popen_persist(commandline).split('\n') + l = [x.split()[-1] for x in l if x] + return l + + def delete(self, filename_list): + """Delete files in filename_list""" + for filename in filename_list: + commandline = "ncftpls -x '' %s -X 'DELE %s' '%s'" % \ + (self.flags, filename, self.url_string) + self.popen_persist(commandline) + + +class rsyncBackend(Backend): + """Connect to remote store using rsync + + rsync backend contributed by Sebastian Wilhelmi + + """ + def __init__(self, parsed_url): + """rsyncBackend initializer""" + Backend.__init__(self, parsed_url) + user, host = parsed_url.netloc.split('@') + if parsed_url.password: + user = user.split(':')[0] + mynetloc = '%s@%s' % (user, host) + # module url: rsync://user@host::/modname/path + # rsync via ssh/rsh: rsync://user@host//some_absolute_path + # -or- rsync://user@host/some_relative_path + if parsed_url.netloc.endswith("::"): + # its a module path + self.url_string = "%s%s" % (mynetloc, parsed_url.path.lstrip('/')) + elif parsed_url.path.startswith("//"): + # its an absolute path + self.url_string = "%s:/%s" % (mynetloc.rstrip(':'), parsed_url.path.lstrip('/')) + else: + # its a relative path + self.url_string = "%s:%s" % (mynetloc.rstrip(':'), parsed_url.path.lstrip('/')) + if self.url_string[-1] != '/': + self.url_string += '/' + + def put(self, source_path, remote_filename = None): + """Use rsync to copy source_dir/filename to remote computer""" + if not remote_filename: remote_filename = source_path.get_filename() + remote_path = os.path.join(self.url_string, remote_filename) + commandline = "rsync %s %s" % (source_path.name, remote_path) + self.run_command(commandline) + + def get(self, remote_filename, local_path): + """Use rsync to get a remote file""" + remote_path = os.path.join (self.url_string, remote_filename) + commandline = "rsync %s %s" % (remote_path, local_path.name) + self.run_command(commandline) + local_path.setdata() + if not local_path.exists(): + raise BackendException("File %s not found" % local_path.name) + + def list(self): + """List files""" + def split (str): + line = str.split () + if len (line) > 4 and line[4] != '.': + return line[4] + else: + return None + commandline = "rsync %s" % self.url_string + return filter(lambda x: x, map (split, self.popen(commandline).split('\n'))) + + def delete(self, filename_list): + """Delete files.""" + delete_list = filename_list + dont_delete_list = [] + for file in self.list (): + if file in delete_list: + delete_list.remove (file) + else: + dont_delete_list.append (file) + if len (delete_list) > 0: + raise BackendException("Files %s not found" % str (delete_list)) + + dir = tempfile.mkdtemp() + exclude, exclude_name = tempdir.default().mkstemp_file() + to_delete = [exclude_name] + for file in dont_delete_list: + path = os.path.join (dir, file) + to_delete.append (path) + f = open (path, 'w') + print >>exclude, file + f.close() + exclude.close() + commandline = ("rsync --recursive --delete --exclude-from=%s %s/ %s" % + (exclude_name, dir, self.url_string)) + self.run_command(commandline) + for file in to_delete: + os.unlink (file) + os.rmdir (dir) + + +class BotoBackend(Backend): + """ + Backend for Amazon's Simple Storage System, (aka Amazon S3), though + the use of the boto module, (http://code.google.com/p/boto/). + + To make use of this backend you must export the environment variables + AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY with your Amazon Web + Services key id and secret respectively. + """ + + def __init__(self, parsed_url): + Backend.__init__(self, parsed_url) + try: + from boto.s3.connection import S3Connection + from boto.s3.key import Key + assert hasattr(S3Connection, 'lookup') + + # Newer versions of boto default to using virtual hosting for + # buckets. This is bad because it will break backups stored in + # buckets that contain upper-case characters in the name. + try: + from boto.s3.connection import OrdinaryCallingFormat + calling_format = OrdinaryCallingFormat() + except ImportError: + calling_format = None + except ImportError: + log.FatalError("This backend requires boto library, version 0.9d or later, " + "(http://code.google.com/p/boto/).") + + if not os.environ.has_key('AWS_ACCESS_KEY_ID'): + raise BackendException("The AWS_ACCESS_KEY_ID environment variable is not set.") + + if not os.environ.has_key('AWS_SECRET_ACCESS_KEY'): + raise BackendException("The AWS_SECRET_ACCESS_KEY environment variable is not set.") + + if parsed_url.scheme == 's3+http': + # Use the default Amazon S3 host. + self.conn = S3Connection() + else: + assert parsed_url.scheme == 's3' + self.conn = S3Connection(host=parsed_url.hostname) + + if hasattr(self.conn, 'calling_format'): + self.conn.calling_format = calling_format + + # This folds the null prefix and all null parts, which means that: + # //MyBucket/ and //MyBucket are equivalent. + # //MyBucket//My///My/Prefix/ and //MyBucket/My/Prefix are equivalent. + self.url_parts = filter(lambda x: x != '', parsed_url.path.split('/')) + + if self.url_parts: + self.bucket_name = self.url_parts.pop(0) + else: + # Duplicity hangs if boto gets a null bucket name. + # HC: Caught a socket error, trying to recover + raise BackendException('Boto requires a bucket name.') + + self.bucket = self.conn.lookup(self.bucket_name) + self.key_class = Key + + if self.url_parts: + self.key_prefix = '%s/' % '/'.join(self.url_parts) + else: + self.key_prefix = '' + + self.straight_url = straight_url(parsed_url) + + + def put(self, source_path, remote_filename=None): + if not self.bucket: + self.bucket = self.conn.create_bucket(self.bucket_name) + if not remote_filename: + remote_filename = source_path.get_filename() + key = self.key_class(self.bucket) + key.key = self.key_prefix + remote_filename + for n in range(1, globals.num_retries+1): + log.Log("Uploading %s/%s" % (self.straight_url, remote_filename), 5) + try: + key.set_contents_from_filename(source_path.name, {'Content-Type': 'application/octet-stream'}) + return + except: + pass + log.Log("Upload '%s/%s' failed (attempt #%d)" % (self.straight_url, remote_filename, n), 1) + time.sleep(30) + log.Log("Giving up trying to upload %s/%s after %d attempts" % (self.straight_url, remote_filename, globals.num_retries), 1) + raise BackendException("Error uploading %s/%s" % (self.straight_url, remote_filename)) + + def get(self, remote_filename, local_path): + key = self.key_class(self.bucket) + key.key = self.key_prefix + remote_filename + for n in range(1, globals.num_retries+1): + log.Log("Downloading %s/%s" % (self.straight_url, remote_filename), 5) + try: + key.get_contents_to_filename(local_path.name) + local_path.setdata() + return + except: + pass + log.Log("Download %s/%s failed (attempt #%d)" % (self.straight_url, remote_filename, n), 1) + time.sleep(30) + log.Log("Giving up trying to download %s/%s after %d attempts" % (self.straight_url, remote_filename, globals.num_retries), 1) + raise BackendException("Error downloading %s/%s" % (self.straight_url, remote_filename)) + + def list(self): + filename_list = [] + if self.bucket: + # We add a 'd' to the prefix to make sure it is not null (for boto) and + # to optimize the listing of our filenames, which always begin with 'd'. + # This will cause a failure in the regression tests as below: + # FAIL: Test basic backend operations + # + # AssertionError: Got list: [] + # Wanted: ['testfile'] + # Because of the need for this optimization, it should be left as is. + #for k in self.bucket.list(prefix = self.key_prefix + 'd', delimiter = '/'): + for k in self.bucket.list(prefix = self.key_prefix, delimiter = '/'): + try: + filename = k.key.replace(self.key_prefix, '', 1) + filename_list.append(filename) + log.Log("Listed %s/%s" % (self.straight_url, filename), 9) + except AttributeError: + pass + return filename_list + + def delete(self, filename_list): + for filename in filename_list: + self.bucket.delete_key(self.key_prefix + filename) + log.Log("Deleted %s/%s" % (self.straight_url, filename), 9) + + +class webdavBackend(Backend): + """Backend for accessing a WebDAV repository. + + webdav backend contributed in 2006 by Jesper Zedlitz + """ + listbody = """\ + + + + + +""" + + """Connect to remote store using WebDAV Protocol""" + def __init__(self, parsed_url): + Backend.__init__(self, parsed_url) + self.headers = {} + self.parsed_url = parsed_url + + + if parsed_url.path: + foldpath = re.compile('/+') + self.directory = foldpath.sub('/', parsed_url.path + '/' ) + else: + self.directory = '/' + + log.Log("Using WebDAV host %s" % (parsed_url.hostname,), 5) + log.Log("Using WebDAV directory %s" % (self.directory,), 5) + log.Log("Using WebDAV protocol %s" % (globals.webdav_proto,), 5) + + password = self.get_password() + + if parsed_url.scheme == 'webdav': + self.conn = httplib.HTTPConnection(parsed_url.hostname) + elif parsed_url.scheme == 'webdavs': + self.conn = httplib.HTTPSConnection(parsed_url.hostname) + else: + raise BackendException("Unknown URI scheme: %s" % (parsed_url.scheme)) + + self.headers['Authorization'] = 'Basic ' + base64.encodestring(parsed_url.username+':'+ password).strip() + + # check password by connection to the server + self.conn.request("OPTIONS", self.directory, None, self.headers) + response = self.conn.getresponse() + response.read() + if response.status != 200: + raise BackendException((response.status, response.reason)) + + def _getText(self,nodelist): + rc = "" + for node in nodelist: + if node.nodeType == node.TEXT_NODE: + rc = rc + node.data + return rc + + def close(self): + self.conn.close() + + def list(self): + """List files in directory""" + for n in range(1, globals.num_retries+1): + log.Log("Listing directory %s on WebDAV server" % (self.directory,), 5) + self.headers['Depth'] = "1" + self.conn.request("PROPFIND", self.directory, self.listbody, self.headers) + del self.headers['Depth'] + response = self.conn.getresponse() + if response.status == 207: + document = response.read() + break + log.Log("WebDAV PROPFIND attempt #%d failed: %s %s" % (n, response.status, response.reason), 5) + if n == globals.num_retries +1: + log.Log("WebDAV backend giving up after %d attempts to PROPFIND %s" % (globals.num_retries, self.directory), 1) + raise BackendException((response.status, response.reason)) + + log.Log("%s" % (document,), 6) + dom = xml.dom.minidom.parseString(document) + result = [] + for href in dom.getElementsByTagName('D:href'): + filename = self.__taste_href(href) + if not filename is None: + result.append(filename) + return result + + def __taste_href(self, href): + """ + Internal helper to taste the given href node and, if + it is a duplicity file, collect it as a result file. + + @returns A matching filename, or None if the href did + not match. + """ + raw_filename = self._getText(href.childNodes).strip() + parsed_url = urlparser.urlparse(urllib.unquote(raw_filename)) + filename = parsed_url.path + log.Debug("webdav path decoding and translation: "\ + "%s -> %s" % (raw_filename, filename)) + + # at least one WebDAV server returns files in the form + # of full URL:s. this may or may not be + # according to the standard, but regardless we + # feel we want to bail out if the hostname + # does not match until someone has looked into + # what the WebDAV protocol mandages. + if not parsed_url.hostname is None \ + and not (parsed_url.hostname == self.parsed_url.hostname): + m = "Received filename was in the form of a "\ + "full url, but the hostname (%s) did "\ + "not match that of the webdav backend "\ + "url (%s) - aborting as a conservative "\ + "safety measure. If this happens to you, "\ + "please report the problem"\ + "" % (parsed_url.hostname, + self.parsed_url.hostname) + raise BackendException(m) + + if filename.startswith(self.directory): + filename = filename.replace(self.directory,'',1) + return filename + else: + return None + + def get(self, remote_filename, local_path): + """Get remote filename, saving it to local_path""" + url = self.directory + remote_filename + target_file = local_path.open("wb") + for n in range(1, globals.num_retries+1): + log.Log("Retrieving %s from WebDAV server" % (url ,), 5) + self.conn.request("GET", url, None, self.headers) + response = self.conn.getresponse() + if response.status == 200: + target_file.write(response.read()) + assert not target_file.close() + local_path.setdata() + return + log.Log("WebDAV GET attempt #%d failed: %s %s" % (n, response.status, response.reason), 5) + log.Log("WebDAV backend giving up after %d attempts to GET %s" % (globals.num_retries, url), 1) + raise BackendException((response.status, response.reason)) + + def put(self, source_path, remote_filename = None): + """Transfer source_path to remote_filename""" + if not remote_filename: + remote_filename = source_path.get_filename() + url = self.directory + remote_filename + source_file = source_path.open("rb") + for n in range(1, globals.num_retries+1): + log.Log("Saving %s on WebDAV server" % (url ,), 5) + self.conn.request("PUT", url, source_file.read(), self.headers) + response = self.conn.getresponse() + if response.status == 201: + response.read() + assert not source_file.close() + return + log.Log("WebDAV PUT attempt #%d failed: %s %s" % (n, response.status, response.reason), 5) + log.Log("WebDAV backend giving up after %d attempts to PUT %s" % (globals.num_retries, url), 1) + raise BackendException((response.status, response.reason)) + + def delete(self, filename_list): + """Delete files in filename_list""" + for filename in filename_list: + url = self.directory + filename + for n in range(1, globals.num_retries+1): + log.Log("Deleting %s from WebDAV server" % (url ,), 5) + self.conn.request("DELETE", url, None, self.headers) + response = self.conn.getresponse() + if response.status == 204: + response.read() + break + log.Log("WebDAV DELETE attempt #%d failed: %s %s" % (n, response.status, response.reason), 5) + if n == globals.num_retries +1: + log.Log("WebDAV backend giving up after %d attempts to DELETE %s" % (globals.num_retries, url), 1) + raise BackendException((response.status, response.reason)) + +hsi_command = "hsi" +class hsiBackend(Backend): + def __init__(self, parsed_url): + Backend.__init__(self, parsed_url) + self.host_string = parsed_url.hostname + self.remote_dir = parsed_url.path + if self.remote_dir: self.remote_prefix = self.remote_dir + "/" + else: self.remote_prefix = "" + + def put(self, source_path, remote_filename = None): + if not remote_filename: remote_filename = source_path.get_filename() + commandline = '%s "put %s : %s%s"' % (hsi_command,source_path.name,self.remote_prefix,remote_filename) + try: + self.run_command(commandline) + except: + print commandline + + def get(self, remote_filename, local_path): + commandline = '%s "get %s : %s%s"' % (hsi_command, local_path.name, self.remote_prefix, remote_filename) + self.run_command(commandline) + local_path.setdata() + if not local_path.exists(): + raise BackendException("File %s not found" % local_path.name) + + def list(self): + commandline = '%s "ls -l %s"' % (hsi_command, self.remote_dir) + l = os.popen3(commandline)[2].readlines()[3:] + for i in range(0,len(l)): + l[i] = l[i].split()[-1] + print filter(lambda x: x, l) + return filter(lambda x: x, l) + + def delete(self, filename_list): + assert len(filename_ist) > 0 + pathlist = map(lambda fn: self.remote_prefix + fn, filename_list) + for fn in filename_list: + commandline = '%s "rm %s%s"' % (hsi_command, self.remote_prefix, fn) + self.run_command(commandline) + +# Dictionary relating protocol strings to backend_object classes. +protocol_class_dict = {"file": LocalBackend, + "ftp": ftpBackend, + "hsi": hsiBackend, + "rsync": rsyncBackend, + "scp": sshBackend, + "ssh": sshBackend, + "s3": BotoBackend, + "s3+http": BotoBackend, + "webdav": webdavBackend, + "webdavs": webdavBackend, + } diff --git a/thirdparty/duplicity/duplicity/collections.py b/thirdparty/duplicity/duplicity/collections.py new file mode 100644 index 000000000..07748c33e --- /dev/null +++ b/thirdparty/duplicity/duplicity/collections.py @@ -0,0 +1,836 @@ +# Copyright 2002 Ben Escoto +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +"""Classes and functions on collections of backup volumes""" + +import gzip, types +import log, file_naming, path, dup_time, globals, manifest + +class CollectionsError(Exception): + pass + +class BackupSet: + """Backup set - the backup information produced by one session""" + def __init__(self, backend): + """Initialize new backup set, only backend is required at first""" + self.backend = backend + self.info_set = None + self.volume_name_dict = {} # dict from volume number to filename + self.remote_manifest_name = None + self.local_manifest_path = None + self.time = None # will be set if is full backup set + self.start_time, self.end_time = None, None # will be set if inc + + def is_complete(self): + """Assume complete if found manifest file""" + return self.remote_manifest_name + + def add_filename(self, filename): + """Add a filename to given set. Return true if it fits. + + The filename will match the given set if it has the right + times and is of the right type. The information will be set + from the first filename given. + + """ + pr = file_naming.parse(filename) + if not pr or not (pr.type == "full" or pr.type == "inc"): + return None + + if not self.info_set: + self.set_info(pr) + else: + if pr.type != self.type: + return None + if pr.time != self.time: + return None + if (pr.start_time != self.start_time or + pr.end_time != self.end_time): + return None + + if pr.manifest: + self.set_manifest(filename) + else: + assert pr.volume_number is not None + assert not self.volume_name_dict.has_key(pr.volume_number), \ + (self.volume_name_dict, filename) + self.volume_name_dict[pr.volume_number] = filename + return 1 + + def set_info(self, pr): + """Set BackupSet information from ParseResults object""" + assert not self.info_set + self.type = pr.type + self.time = pr.time + self.start_time, self.end_time = pr.start_time, pr.end_time + self.time = pr.time + self.info_set = 1 + + def set_manifest(self, remote_filename): + """Add local and remote manifest filenames to backup set""" + assert not self.remote_manifest_name, (self.remote_manifest_name, + remote_filename) + self.remote_manifest_name = remote_filename + + if not globals.archive_dir: + return + for local_filename in globals.archive_dir.listdir(): + pr = file_naming.parse(local_filename) + if (pr and pr.manifest and pr.type == self.type and + pr.time == self.time and pr.start_time == self.start_time + and pr.end_time == self.end_time): + self.local_manifest_path = \ + globals.archive_dir.append(local_filename) + break + + def delete(self): + """Remove all files in set""" + l = self.get_filenames() + l.reverse() # delete starting with manifest + self.backend.delete(l) + + def __str__(self): + """For now just list files in set""" + filelist = [] + if self.remote_manifest_name: + filelist.append(self.remote_manifest_name) + filelist.extend(self.volume_name_dict.values()) + return "[%s]" % ", ".join(filelist) + + def get_timestr(self): + """Return time string suitable for log statements""" + return dup_time.timetopretty(self.time or self.end_time) + + def check_manifests(self): + """Make sure remote manifest is equal to local one""" + if not self.remote_manifest_name and not self.local_manifest_path: + log.FatalError("Fatal Error: No manifests found for most recent backup") + assert self.remote_manifest_name, "if only one, should be remote" + + remote_manifest = self.get_remote_manifest() + if self.local_manifest_path: + local_manifest = self.get_local_manifest() + if remote_manifest and self.local_manifest_path and local_manifest: + if remote_manifest != local_manifest: + log.FatalError("Fatal Error: Remote manifest does not match local one. Either the " + "remote backup set or the local archive directory has been corrupted.") + if not remote_manifest: + if self.local_manifest_path: + remote_manifest = local_manifest + else: + log.FatalError("Fatal Error: Neither remote nor local manifest is readable.") + remote_manifest.check_dirinfo() + + def get_local_manifest(self): + """Return manifest object by reading local manifest file""" + assert self.local_manifest_path + manifest_buffer = self.local_manifest_path.get_data() + return manifest.Manifest().from_string(manifest_buffer) + + def get_remote_manifest(self): + """Return manifest by reading remote manifest on backend""" + assert self.remote_manifest_name + # Following by MDR. Should catch if remote encrypted with + # public key w/o secret key + try: + manifest_buffer = self.backend.get_data(self.remote_manifest_name) + except IOError, message: + if message.args[0] == "GnuPG exited non-zero, with code 131072": + return None + else: + raise + return manifest.Manifest().from_string(manifest_buffer) + + def get_manifest(self): + """Return manifest object, showing preference for local copy""" + if self.local_manifest_path: + return self.get_local_manifest() + else: + return self.get_remote_manifest() + + def get_filenames(self): + """Return sorted list of (remote) filenames of files in set""" + assert self.info_set + volume_num_list = self.volume_name_dict.keys() + volume_num_list.sort() + volume_filenames = map(lambda x: self.volume_name_dict[x], + volume_num_list) + if self.remote_manifest_name: + return [self.remote_manifest_name] + volume_filenames + else: + return volume_filenames + + def get_time(self): + """Return time if full backup, or end_time if incremental""" + if self.time: + return self.time + if self.end_time: + return self.end_time + assert 0, "Neither self.time nor self.end_time set" + + def __len__(self): + """Return the number of volumes in the set""" + return len(self.volume_name_dict.keys()) + + +class BackupChain: + """BackupChain - a number of linked BackupSets + + A BackupChain always starts with a full backup set and continues + with incremental ones. + + """ + def __init__(self, backend): + """Initialize new chain, only backend is required at first""" + self.backend = backend + self.fullset = None + self.incset_list = [] # sorted list of BackupSets + self.start_time, self.end_time = None, None + + def set_full(self, fullset): + """Add full backup set""" + assert not self.fullset and isinstance(fullset, BackupSet) + self.fullset = fullset + assert fullset.time + self.start_time, self.end_time = fullset.time, fullset.time + + def add_inc(self, incset): + """Add incset to self. Return None if incset does not match""" + if self.end_time == incset.start_time: + self.incset_list.append(incset) + else: + if (self.incset_list + and incset.start_time == self.incset_list[-1].start_time + and incset.end_time > self.incset_list[-1]): + log.Log("Preferring Backupset over previous one!", 8) + self.incset_list[-1] = incset + else: + log.Log("Ignoring incremental Backupset (start_time: %s; needed: %s)" % + (dup_time.timetopretty(incset.start_time), + dup_time.timetopretty(self.end_time)), 8) + return None + self.end_time = incset.end_time + log.Log("Added incremental Backupset (start_time: %s / end_time: %s)" % + (dup_time.timetopretty(incset.start_time), + dup_time.timetopretty(incset.end_time)), 8) + assert self.end_time + return 1 + + def delete(self): + """Delete all sets in chain, in reverse order""" + for i in range(len(self.incset_list)-1, -1, -1): + self.incset_list[i].delete() + if self.fullset: + self.fullset.delete() + + def get_sets_at_time(self, time): + """Return a list of sets in chain earlier or equal to time""" + older_incsets = filter(lambda s: s.end_time <= time, self.incset_list) + return [self.fullset] + older_incsets + + def get_last(self): + """Return last BackupSet in chain""" + if self.incset_list: + return self.incset_list[-1] + else: + return self.fullset + + def get_first(self): + """Return first BackupSet in chain (ie the full backup)""" + return self.fullset + + def short_desc(self): + """Return a short one-line description of the chain, suitable + for log messages.""" + return "[%s]-[%s]" % (dup_time.timetopretty(self.start_time), + dup_time.timetopretty(self.end_time)) + + def __str__(self): + """Return string representation, for testing purposes""" + set_schema = "%20s %30s %15s" + l = ["-------------------------", + "Chain start time: " + dup_time.timetopretty(self.start_time), + "Chain end time: " + dup_time.timetopretty(self.end_time), + "Number of contained backup sets: %d" % + (len(self.incset_list)+1,), + "Total number of contained volumes: %d" % + (self.get_num_volumes(),), + set_schema % ("Type of backup set:", "Time:", "Num volumes:")] + + for s in self.get_all_sets(): + if s.time: + type = "Full" + time = s.time + else: + type = "Incremental" + time = s.end_time + l.append(set_schema % (type, dup_time.timetopretty(time), len(s))) + + l.append("-------------------------") + return "\n".join(l) + + def get_num_volumes(self): + """Return the total number of volumes in the chain""" + n = 0 + for s in self.get_all_sets(): + n += len(s) + return n + + def get_all_sets(self): + """Return list of all backup sets in chain""" + if self.fullset: + return [self.fullset] + self.incset_list + else: + return self.incset_list + + +class SignatureChain: + """A number of linked signatures + + Analog to BackupChain - start with a full-sig, and continue with + new-sigs. + + """ + def __init__(self, local, location): + """Return new SignatureChain. + + local should be true iff the signature chain resides in + globals.archive_dir and false if the chain is in + globals.backend. + + """ + if local: + self.archive_dir, self.backend = location, None + else: + self.archive_dir, self.backend = None, location + self.fullsig = None # filename of full signature + self.inclist = [] # list of filenames of incremental signatures + self.start_time, self.end_time = None, None + + def __str__(self): + """Local or Remote and List of files in the set""" + if self.archive_dir: + place = "local" + else: + place = "remote" + filelist = [] + if self.fullsig: + filelist.append(self.fullsig) + filelist.extend(self.inclist) + return "%s: [%s]" % (place, ", ".join(filelist)) + + def check_times(self, time_list): + """Check to make sure times are in whole seconds""" + for time in time_list: + if type(time) not in (types.LongType, types.IntType): + assert 0, "Time %s in %s wrong type" % (time, time_list) + + def islocal(self): + """Return true if represents a signature chain in archive_dir""" + if self.archive_dir: + return True + else: + return False + + def add_filename(self, filename, pr = None): + """Add new sig filename to current chain. Return true if fits""" + if not pr: + pr = file_naming.parse(filename) + if not pr: + return None + + if self.fullsig: + if pr.type != "new-sig": + return None + if pr.start_time != self.end_time: + return None + self.inclist.append(filename) + self.check_times([pr.end_time]) + self.end_time = pr.end_time + return 1 + else: + if pr.type != "full-sig": + return None + self.fullsig = filename + self.check_times([pr.time, pr.time]) + self.start_time, self.end_time = pr.time, pr.time + return 1 + + def get_fileobjs(self): + """Return ordered list of signature fileobjs opened for reading""" + assert self.fullsig + if self.archive_dir: # local + def filename_to_fileobj(filename): + """Open filename in archive_dir, return filtered fileobj""" + sig_dp = path.DupPath(self.archive_dir.name, (filename,)) + return sig_dp.filtered_open("rb") + else: + filename_to_fileobj = self.backend.get_fileobj_read + return map(filename_to_fileobj, [self.fullsig] + self.inclist) + + def delete(self): + """Remove all files in signature set""" + # Try to delete in opposite order, so something useful even if aborted + if self.archive_dir: + for i in range(len(self.inclist)-1, -1, -1): + self.archive_dir.append(self.inclist[i]).delete() + self.archive_dir.append(self.fullsig).delete() + else: + assert self.backend + inclist_copy = self.inclist[:] + inclist_copy.reverse() + inclist_copy.append(self.fullsig) + self.backend.delete(inclist_copy) + + def get_filenames(self): + """Return ordered list of filenames in set""" + if self.fullsig: + l = [self.fullsig] + else: + l = [] + l.extend(self.inclist) + return l + + +class CollectionsStatus: + """Hold information about available chains and sets""" + def __init__(self, backend, archive_dir = None): + """Make new object. Does not set values""" + self.backend, self.archive_dir = backend, archive_dir + + # Will hold (signature chain, backup chain) pair of active + # (most recent) chains + self.matched_chain_pair = None + + # These should be sorted by end_time + self.all_backup_chains = None + self.other_backup_chains = None + self.other_sig_chains = None + + # Other misc paths and sets which shouldn't be there + self.orphaned_sig_names = None + self.orphaned_backup_sets = None + self.incomplete_backup_sets = None + + # True if set_values() below has run + self.values_set = None + + def __str__(self): + """Return string summary of the collection""" + l = ["Connecting with backend: %s" % + (self.backend.__class__.__name__,), + "Archive dir: %s" % (self.archive_dir,)] + + l.append("\nFound %d backup chains without signatures." + % len(self.other_backup_chains)) + for i in range(len(self.other_backup_chains)): + l.append("Signature-less chain %d of %d:" % + (i+1, len(self.other_backup_chains))) + l.append(str(self.other_backup_chains[i])) + l.append("") + + if self.matched_chain_pair: + l.append("\nFound a complete backup chain with matching " + "signature chain:") + l.append(str(self.matched_chain_pair[1])) + else: + l.append("No backup chains with active signatures found") + + if self.orphaned_backup_sets or self.incomplete_backup_sets: + l.append("Also found %d backup sets not part of any chain," + % (len(self.orphaned_backup_sets),)) + l.append("and %d incomplete backup sets." + % (len(self.incomplete_backup_sets),)) + l.append("These may be deleted by running duplicity with the " + "--cleanup option.") + else: + l.append("No orphaned or incomplete backup sets found.") + + return "\n".join(l) + + def set_values(self, sig_chain_warning = 1): + """Set values from archive_dir and backend. + + if archive_dir is None, omit any local chains. Returns self + for convenience. If sig_chain_warning is set to None, do not + warn about unnecessary sig chains. This is because there may + naturally be some unecessary ones after a full backup. + + """ + self.values_set = 1 + backend_filename_list = self.backend.list() + log.Debug("%d files exist on backend" % (len(backend_filename_list,))) + + (backup_chains, self.orphaned_backup_sets, + self.incomplete_backup_sets) = \ + self.get_backup_chains(backend_filename_list) + backup_chains = self.get_sorted_chains(backup_chains) + self.all_backup_chains = backup_chains + + assert len(backup_chains) == len(self.all_backup_chains), "get_sorted_chains() did something more than re-ordering" + + if self.archive_dir: + local_sig_chains, local_orphaned_sig_names = \ + self.get_signature_chains(True) + else: + local_sig_chains, local_orphaned_sig_names = [], [] + remote_sig_chains, remote_orphaned_sig_names = \ + self.get_signature_chains(False, filelist = backend_filename_list) + self.orphaned_sig_names = (local_orphaned_sig_names + + remote_orphaned_sig_names) + self.set_matched_chain_pair(local_sig_chains + remote_sig_chains, + backup_chains) + self.warn(sig_chain_warning) + return self + + def set_matched_chain_pair(self, sig_chains, backup_chains): + """Set self.matched_chain_pair and self.other_sig/backup_chains + + The latest matched_chain_pair will be set. If there are both + remote and local signature chains capable of matching the + latest backup chain, use the local sig chain (it does not need + to be downloaded). + + """ + self.other_sig_chains = sig_chains + self.other_backup_chains = backup_chains[:] + self.matched_chain_pair = None + if sig_chains and backup_chains: + latest_backup_chain = backup_chains[-1] + sig_chains = self.get_sorted_chains(sig_chains) + for i in range(len(sig_chains)-1, -1, -1): + if sig_chains[i].end_time == latest_backup_chain.end_time: + pass + # See if the set before last matches: + elif (len(latest_backup_chain.get_all_sets()) >= 2 and + sig_chains[i].end_time == latest_backup_chain.get_all_sets()[-2].end_time): + # It matches, remove the last backup set: + log.Warn("Warning, discarding last backup set, because of missing signature file.") + self.incomplete_backup_sets.append(latest_backup_chain.incset_list[-1]) + latest_backup_chain.incset_list = latest_backup_chain.incset_list[:-1] + else: + continue + + # Found a matching pair: + if self.matched_chain_pair == None: + self.matched_chain_pair = (sig_chains[i], latest_backup_chain) + + del sig_chains[i] + break + + if self.matched_chain_pair: + # if we have local and remote sig chains, remove both from the other_sig_chains list + matched_sig_chain = self.matched_chain_pair[0] + if len(self.other_sig_chains) > 1: + for sig_chain in self.other_sig_chains[1:]: + if (sig_chain.islocal() != matched_sig_chain.islocal() and + sig_chain.start_time == matched_sig_chain.start_time and + sig_chain.end_time == matched_sig_chain.end_time): + self.other_sig_chains.remove(sig_chain) + self.other_sig_chains.remove(matched_sig_chain) + self.other_backup_chains.remove(self.matched_chain_pair[1]) + + def warn(self, sig_chain_warning): + """Log various error messages if find incomplete/orphaned files""" + assert self.values_set + if self.orphaned_sig_names: + log.Log("Warning, found the following orphaned signature files:\n" + + "\n".join(self.orphaned_sig_names), 2) + if self.other_sig_chains and sig_chain_warning: + if self.matched_chain_pair: + log.Log("Warning, found unnecessary signature chain(s)", 2) + else: + log.Log("Warning, found signatures but no corresponding " + "backup files", 2) + + if self.incomplete_backup_sets: + log.Log("Warning, found incomplete backup sets, probably left " + "from aborted session", 2) + if self.orphaned_backup_sets: + log.Log("Warning, found the following orphaned backup files:\n" + + "\n".join(map(lambda x: str(x), + self.orphaned_backup_sets)), 2) + + def get_backup_chains(self, filename_list): + """Split given filename_list into chains + + Return value will be pair (list of chains, list of sets, list + of incomplete sets), where the list of sets will comprise sets + not fitting into any chain, and the incomplete sets are sets + missing files. + + """ + log.Debug("Extracting backup chains from list of files: %s" % (filename_list,)) + # First put filenames in set form + sets = [] + def add_to_sets(filename): + """Try adding filename to existing sets, or make new one""" + for set in sets: + if set.add_filename(filename): + log.Debug("File %s is part of known set" % (filename,)) + break + else: + log.Debug("File %s is not part of a known set; creating new set" % (filename,)) + new_set = BackupSet(self.backend) + if new_set.add_filename(filename): + sets.append(new_set) + else: + log.Log("Ignoring file (rejected by backup set) '%s'" % filename, 9) + map(add_to_sets, filename_list) + sets, incomplete_sets = self.get_sorted_sets(sets) + + chains, orphaned_sets = [], [] + def add_to_chains(set): + """Try adding set to existing chains, or make new one""" + if set.type == "full": + new_chain = BackupChain(self.backend) + new_chain.set_full(set) + chains.append(new_chain) + log.Debug("Found backup chain %s" % (new_chain.short_desc())) + else: + assert set.type == "inc" + for chain in chains: + if chain.add_inc(set): + log.Debug("Added set %s to pre-existing chain %s" % (set.get_timestr(), + chain.short_desc())) + break + else: + log.Debug("Found orphaned set %s" % (set.get_timestr(),)) + orphaned_sets.append(set) + map(add_to_chains, sets) + return (chains, orphaned_sets, incomplete_sets) + + def get_sorted_sets(self, set_list): + """Sort set list by end time, return (sorted list, incomplete)""" + time_set_pairs, incomplete_sets = [], [] + for set in set_list: + if not set.is_complete(): + incomplete_sets.append(set) + elif set.type == "full": + time_set_pairs.append((set.time, set)) + else: + time_set_pairs.append((set.end_time, set)) + time_set_pairs.sort() + return (map(lambda p: p[1], time_set_pairs), incomplete_sets) + + def get_signature_chains(self, local, filelist = None): + """Find chains in archive_dir (if local is true) or backend + + Use filelist if given, otherwise regenerate. Return value is + pair (list of chains, list of signature paths not in any + chains). + + """ + def get_filelist(): + if filelist is not None: + return filelist + elif local: + return self.archive_dir.listdir() + else: + return self.backend.list() + + def get_new_sigchain(): + """Return new empty signature chain""" + if local: + return SignatureChain(True, self.archive_dir) + else: + return SignatureChain(False, self.backend) + + # Build initial chains from full sig filenames + chains, new_sig_filenames = [], [] + for filename in get_filelist(): + pr = file_naming.parse(filename) + if pr: + if pr.type == "full-sig": + new_chain = get_new_sigchain() + assert new_chain.add_filename(filename, pr) + chains.append(new_chain) + elif pr.type == "new-sig": + new_sig_filenames.append(filename) + + # Try adding new signatures to existing chains + orphaned_filenames = [] + new_sig_filenames.sort() + for sig_filename in new_sig_filenames: + for chain in chains: + if chain.add_filename(sig_filename): + break + else: + orphaned_filenames.append(sig_filename) + return (chains, orphaned_filenames) + + def get_sorted_chains(self, chain_list): + """Return chains sorted by end_time. If tie, local goes last""" + # Build dictionary from end_times to lists of corresponding chains + endtime_chain_dict = {} + for chain in chain_list: + if endtime_chain_dict.has_key(chain.end_time): + endtime_chain_dict[chain.end_time].append(chain) + else: + endtime_chain_dict[chain.end_time] = [chain] + + # Use dictionary to build final sorted list + sorted_end_times = endtime_chain_dict.keys() + sorted_end_times.sort() + sorted_chain_list = [] + for end_time in sorted_end_times: + chain_list = endtime_chain_dict[end_time] + if len(chain_list) == 1: + sorted_chain_list.append(chain_list[0]) + else: + assert len(chain_list) == 2 + if chain_list[0].backend: # is remote, goes first + assert chain_list[1].archive_dir # other is local + sorted_chain_list.append(chain_list[0]) + sorted_chain_list.append(chain_list[1]) + else: # is local, goes second + assert chain_list[1].backend # other is remote + sorted_chain_list.append(chain_list[1]) + sorted_chain_list.append(chain_list[0]) + + return sorted_chain_list + + def get_backup_chain_at_time(self, time): + """Return backup chain covering specified time + + Tries to find the backup chain covering the given time. If + there is none, return the earliest chain before, and failing + that, the earliest chain. + + """ + if not self.all_backup_chains: + raise CollectionsError("No backup chains found") + + covering_chains = filter(lambda c: c.start_time <= time <= c.end_time, + self.all_backup_chains) + if len(covering_chains) > 1: + raise CollectionsError("Two chains cover the given time") + elif len(covering_chains) == 1: + return covering_chains[0] + + old_chains = filter(lambda c: c.end_time < time, + self.all_backup_chains) + if old_chains: + return old_chains[-1] + else: + return self.all_backup_chains[0] # no chains are old enough + + def cleanup_signatures(self): + """Delete unnecessary older signatures""" + map(SignatureChain.delete, self.other_sig_chains) + + def get_extraneous(self): + """Return list of the names of extraneous duplicity files + + A duplicity file is considered extraneous if it is + recognizable as a duplicity file, but isn't part of some + complete backup set, or current signature chain. + + """ + assert self.values_set + filenames = [] + ext_containers = (self.other_sig_chains, self.orphaned_backup_sets, + self.incomplete_backup_sets) + for set_or_chain_list in ext_containers: + for set_or_chain in set_or_chain_list: + filenames.extend(set_or_chain.get_filenames()) + filenames.extend(self.orphaned_sig_names) + return filenames + + def sort_sets(self, setlist): + """Return new list containing same elems of setlist, sorted by time""" + pairs = map(lambda s: (s.get_time(), s), setlist) + pairs.sort() + return map(lambda p: p[1], pairs) + + def get_chains_older_than(self, t): + """Return a list of chains older than time t""" + assert self.values_set + return filter(lambda c: c.end_time < t, self.all_backup_chains) + + def get_last_full_backup_time(self): + """Return the time of the last full backup, or 0 if + there is none.""" + return self.get_nth_last_full_backup_time(1) + + def get_nth_last_full_backup_time(self, n): + """Return the time of the nth to last full backup, or 0 + if there is none.""" + chain = self.get_nth_last_backup_chain(n) + if chain is None: + return 0 + else: + return chain.get_first().time + + def get_last_backup_chain(self): + """Return the last full backup of the collection, or None + if there is no full backup chain.""" + return self.get_nth_last_backup_chain(1) + + def get_nth_last_backup_chain(self,n): + """Return the nth-to-last full backup of the collection, or None + if there is less than n backup chains. + + NOTE: n = 1 -> time of latest available chain (n = 0 is not + a valid input). Thus the second-to-last is obtained with n=2 + rather than n=1.""" + assert self.values_set + assert n > 0 + + if len(self.all_backup_chains) < n: + return None + + sorted = self.all_backup_chains[:] + sorted.sort(reverse = True, + key = lambda chain: chain.get_first().time) + + return sorted[n - 1] + + def get_older_than(self, t): + """Returns a list of backup sets older than the given time t + + All of the times will be associated with an intact chain. + Furthermore, none of the times will be of a set which a newer + set may depend on. For instance, if set A is a full set older + than t, and set B is an incremental based on A which is newer + than tt, then the time of set A will not be returned. + + """ + old_sets = [] + for chain in self.get_chains_older_than(t): + if (not self.matched_chain_pair or + chain is not self.matched_chain_pair[1]): + # don't delete the active (matched) chain + old_sets.extend(chain.get_all_sets()) + return self.sort_sets(old_sets) + + def get_older_than_required(self, t): + """Returns list of old backup sets required by new sets + + This function is similar to the previous one, but it only + returns the times of sets which are old but part of the chains + where the newer end of the chain is newer than t. + + """ + assert self.values_set + new_chains = filter(lambda c: c.end_time >= t, self.all_backup_chains) + result_sets = [] + for chain in new_chains: + old_sets = filter(lambda s: s.get_time() < t, chain.get_all_sets()) + result_sets.extend(old_sets) + return self.sort_sets(result_sets) diff --git a/thirdparty/duplicity/duplicity/commandline.py b/thirdparty/duplicity/duplicity/commandline.py new file mode 100644 index 000000000..f8b3f59dd --- /dev/null +++ b/thirdparty/duplicity/duplicity/commandline.py @@ -0,0 +1,476 @@ +# Copyright 2002 Ben Escoto +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 3 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +"""Parse command line, check for consistency, and set globals""" + +import getopt, re, sys, os +import backends, globals, log, path, selection, gpg, dup_time + +select_opts = [] # Will hold all the selection options +select_files = [] # Will hold file objects when filelist given +full_backup = None # Will be set to true if -f or --full option given +list_current = None # Will be set to true if --list-current option given +collection_status = None # Will be set to true if --collection-status given +cleanup = None # Set to true if --cleanup option given +verify = None # Set to true if --verify option given + +commands = ["cleanup", + "collection-status", + "full", + "incremental", + "list-current-files", + "remove-older-than", + "remove-all-but-n-full", + "restore", + "verify", + ] + +options = ["allow-source-mismatch", + "archive-dir=", + "current-time=", + "encrypt-key=", + "exclude=", + "exclude-device-files", + "exclude-filelist=", + "exclude-globbing-filelist=", + "exclude-filelist-stdin", + "exclude-other-filesystems", + "exclude-regexp=", + "file-to-restore=", + "force", + "ftp-passive", + "ftp-regular", + "full-if-older-than=", + "gpg-options=", + "help", + "include=", + "include-filelist=", + "include-filelist-stdin", + "include-globbing-filelist=", + "include-regexp=", + "no-encryption", + "no-print-statistics", + "null-separator", + "num-retries=", + "restore-dir=", + "restore-time=", + "scp-command=", + "sftp-command=", + "short-filenames", + "sign-key=", + "ssh-askpass", + "ssh-options=", + "tempdir=", + "timeout=", + "time-separator=", + "verbosity=", + "version", + "volsize=", + ] + +def parse_cmdline_options(arglist): + """Parse argument list""" + global select_opts, select_files, full_backup + global list_current, collection_status, cleanup, remove_time, verify + + def sel_fl(filename): + """Helper function for including/excluding filelists below""" + try: + return open(filename, "r") + except IOError: + log.FatalError("Error opening file %s" % filename) + + # expect no cmd and two positional args + cmd = "" + num_expect = 2 + + # process first arg as command + if arglist and arglist[0][0] != '-': + cmd = arglist.pop(0) + possible = [c for c in commands if c.startswith(cmd)] + # no unique match, that's an error + if len(possible) > 1: + log.FatalError("command '%s' not unique, could be %s" % (cmd, possible)) + # only one match, that's a keeper + elif len(possible) == 1: + cmd = possible[0] + # no matches, assume no cmd + elif not possible: + arglist.insert(0, cmd) + + if cmd == "cleanup": + cleanup = True + num_expect = 1 + elif cmd == "collection-status": + collection_status = True + num_expect = 1 + elif cmd == "full": + full_backup = True + num_expect = 2 + elif cmd == "incremental": + globals.incremental = True + num_expect = 2 + elif cmd == "list-current-files": + list_current = True + num_expect = 1 + elif cmd == "remove-older-than": + try: + arg = arglist.pop(0) + except: + command_line_error("Missing time string for remove-older-than") + globals.remove_time = dup_time.genstrtotime(arg) + num_expect = 1 + elif cmd == "remove-all-but-n-full": + try: + arg = arglist.pop(0) + except: + command_line_error("Missing count for remove-all-but-n-full") + globals.keep_chains = int(arg) + + if not globals.keep_chains > 0: + command_line_error("remove-all-but-n-full count must be > 0") + + num_expect = 1 + elif cmd == "verify": + verify = True + num_expect = 2 + + # parse the remaining args + try: + optlist, args = getopt.gnu_getopt(arglist, "rt:v:V", options) + except getopt.error, e: + command_line_error("%s" % (str(e),)) + + for opt, arg in optlist: + if opt == "--allow-source-mismatch": + globals.allow_source_mismatch = 1 + elif opt == "--archive-dir": + set_archive_dir(arg) + elif opt == "--current-time": + dup_time.setcurtime(get_int(arg, "current-time")) + elif opt == "--encrypt-key": + globals.gpg_profile.recipients.append(arg) + elif (opt == "--exclude" or + opt == "--exclude-regexp" or + opt == "--include" or + opt == "--include-regexp"): + select_opts.append((opt, arg)) + elif (opt == "--exclude-device-files" or + opt == "--exclude-other-filesystems"): + select_opts.append((opt, None)) + elif (opt == "--exclude-filelist" or + opt == "--include-filelist" or + opt == "--exclude-globbing-filelist" or + opt == "--include-globbing-filelist"): + select_opts.append((opt, arg)) + select_files.append(sel_fl(arg)) + elif opt == "--exclude-filelist-stdin": + select_opts.append(("--exclude-filelist", "standard input")) + select_files.append(sys.stdin) + elif opt == "--full-if-older-than": + globals.full_force_time = dup_time.genstrtotime(arg) + elif opt == "--force": + globals.force = 1 + elif opt == "--ftp-passive": + globals.ftp_connection = 'passive' + elif opt == "--ftp-regular": + globals.ftp_connection = 'regular' + elif opt == "--gpg-options": + gpg.gpg_options = (gpg.gpg_options + ' ' + arg).strip() + elif opt == "--help": + usage(); + sys.exit(1); + elif opt == "--include-filelist-stdin": + select_opts.append(("--include-filelist", "standard input")) + select_files.append(sys.stdin) + elif opt == "--no-encryption": + globals.encryption = 0 + elif opt == "--no-print-statistics": + globals.print_statistics = 0 + elif opt == "--null-separator": + globals.null_separator = 1 + elif opt == "--num-retries": + globals.num_retries = int(arg) + elif (opt == "-r" or + opt == "--file-to-restore"): + globals.restore_dir = arg + elif (opt == "-t" or + opt == "--restore-time"): + globals.restore_time = dup_time.genstrtotime(arg) + elif opt == "--scp-command": + backends.scp_command = arg + elif opt == "--sftp-command": + backends.sftp_command = arg + elif opt == "--short-filenames": + globals.short_filenames = 1 + elif opt == "--sign-key": + set_sign_key(arg) + elif opt == "--ssh-askpass": + backends.ssh_askpass = True + elif opt == "--ssh-options": + backends.ssh_options = (backends.ssh_options + ' ' + arg).strip() + elif opt == "--tempdir": + globals.temproot = arg + elif opt == "--timeout": + globals.timeout = int(arg) + elif opt == "--time-separator": + if arg == '-': + command_line_error("Dash ('-') not valid for time-separator.") + globals.time_separator = arg + dup_time.curtimestr = dup_time.timetostring(dup_time.curtime) + elif opt == "-V" or opt == "--version": + print "duplicity", str(globals.version) + sys.exit(0) + elif (opt == "-v" or + opt == "--verbosity"): + log.setverbosity(int(arg)) + elif opt == "--volsize": + globals.volsize = int(arg)*1024*1024 + else: + command_line_error("Unknown option %s" % opt) + + if len(args) != num_expect: + command_line_error("Expected %d args, got %d" % (num_expect, len(args))) + + return args + +def command_line_error(message): + """Indicate a command line error and exit""" + sys.stderr.write("Command line error: %s\n" % (message,)) + sys.stderr.write("Enter 'duplicity --help' for help screen.\n") + sys.exit(1) + +def usage(): + """Print terse usage info""" + sys.stdout.write(""" +duplicity version %s running on %s. +Usage: + duplicity [full|incremental] [options] source_dir target_url + duplicity [restore] [options] source_url target_dir + duplicity verify [options] source_url target_dir + duplicity collection-status [options] target_url + duplicity list-current-files [options] target_url + duplicity cleanup [options] target_url + duplicity remove-older-than time [options] target_url + duplicity remove-all-but-n-full count [options] target_url + +Backends and their URL formats: + ssh://user[:password]@other.host[:port]/some_dir + scp://user[:password]@other.host[:port]/some_dir + ftp://user[:password]@other.host[:port]/some_dir + hsi://user[:password]@other.host[:port]/some_dir + file:///some_dir + rsync://user[:password]@other.host[:port]::/module/some_dir + rsync://user[:password]@other.host[:port]/relative_path + rsync://user[:password]@other.host[:port]//absolute_path + s3://other.host/bucket_name[/prefix] + s3+http://bucket_name[/prefix] + webdav://user[:password]@other.host/some_dir + webdavs://user[:password]@other.host/some_dir + +Commands: + cleanup + collection-status + full + incr + list-current-files + restore + remove-older-than