diff --git a/blueprint/io/__init__.py b/blueprint/io/__init__.py index 899d797..55adb4b 100644 --- a/blueprint/io/__init__.py +++ b/blueprint/io/__init__.py @@ -14,7 +14,6 @@ def pull(server, secret, name): r = http.get('/{0}/{1}'.format(secret, name), server=server) if 200 == r.status: b = Blueprint.load(r, name) - for filename in b.sources.itervalues(): logging.info('fetching source tarballs - this may take a while') r = http.get('/{0}/{1}/{2}'.format(secret, name, filename), diff --git a/blueprint/io/server/__init__.py b/blueprint/io/server/__init__.py index a7b6194..f35efa6 100644 --- a/blueprint/io/server/__init__.py +++ b/blueprint/io/server/__init__.py @@ -9,13 +9,18 @@ from blueprint import Blueprint from blueprint import cfg -import backend + import librato import statsd +backend = cfg.get('server', 'backend') +if backend == "mongodb": + import backend_mongodb as backend +else: + import backend -app = Flask(__name__) +app = Flask(__name__) def _blueprint(secret, name): """ @@ -114,6 +119,12 @@ def secret(): browser_pattern = re.compile(r'Chrome|Gecko|Microsoft|Mozilla|Safari|WebKit') +@app.route('///blueprint.json', methods=['GET']) +def get_json(secret, name): + validate_secret(secret) + validate_name(name) + + return backend.get(backend.key_for_blueprint(secret, name)) @app.route('//', methods=['GET']) def get_blueprint(secret, name): diff --git a/blueprint/io/server/backend_mongodb.py b/blueprint/io/server/backend_mongodb.py new file mode 100644 index 0000000..59b1d22 --- /dev/null +++ b/blueprint/io/server/backend_mongodb.py @@ -0,0 +1,166 @@ +from pymongo import MongoClient +import httplib +import socket + +from blueprint import cfg +import librato +import statsd + +username = None +password = None + +address = cfg.get('mongodb', 'address') +port = cfg.get('mongodb', 'port') +database = cfg.get('mongodb', 'database') +collection = cfg.get('mongodb', 'collection') +url = cfg.get('server', 'address') +protocol = 'https' if cfg.getboolean('server', 'use_https') else 'http' +try: + username = cfg.get('mongodb', 'user') + password = cfg.get('mongodb', 'password') +except: + pass + +client = MongoClient(address, int(port)) +db = client[database] +if username is not None: + if password is not None: + db.authenticate(username, password) +collection = db[collection] + +class StoredObject: + key = None + tarball = None + def __init__(self, key, tarball): + self.key = key + self.tarball = tarball + +def delete(key): + """ + Remove an object from MongoDB. + """ + content_length = head(key) + if content_length is None: + return None + librato.count('blueprint-io-server.requests.delete') + statsd.increment('blueprint-io-server.requests.delete') + try: + collection.delete({"key" : key}) + except: + return False + +def delete_blueprint(secret, name): + return delete(key_for_blueprint(secret, name)) + + +def delete_tarball(secret, name, sha): + return delete(key_for_tarball(secret, name, sha)) + + +def get(key): + """ + Fetch an object from MongoDB. + """ + librato.count('blueprint-io-server.requests.get') + statsd.increment('blueprint-io-server.requests.get') + try: + k = collection.find_one({"key" : key}) + if k is None: + return None + except: + return False + return k['tarball'] + + +def get_blueprint(secret, name): + return get(key_for_blueprint(secret, name)) + + +def get_tarball(secret, name, sha): + return get(key_for_tarball(secret, name, sha)) + + +def head(key): + """ + Make a HEAD request for an object in MongoDB. + This returns the size of the tarball. + """ + librato.count('blueprint-io-server.requests.head') + statsd.increment('blueprint-io-server.requests.head') + try: + k = collection.find_one({"key" : key}) + if k is None: + return None + except: + return False + return len(k['tarball']) + + +def head_blueprint(secret, name): + return head(key_for_blueprint(secret, name)) + + +def head_tarball(secret, name, sha): + return head(key_for_tarball(secret, name, sha)) + + +def key_for_blueprint(secret, name): + return '{0}/{1}/{2}'.format(secret, + name, + 'blueprint.json') + + +def key_for_tarball(secret, name, sha): + return '{0}/{1}/{2}.tar'.format(secret, + name, + sha) + + +def list(key): + """ + List objects in MongoDB whose key begins with the given prefix + """ + librato.count('blueprint-io-server.requests.list') + statsd.increment('blueprint-io-server.requests.list') + try: + result = collection.find({"key" : '^%s' % (key)}) + except: + return False + return result + + +def put(key, data): + """ + Store an object in MongoDB. + """ + librato.count('blueprint-io-server.requests.put') + statsd.increment('blueprint-io-server.requests.put') + # TODO librato.something('blueprint-io-server.storage', len(data)) + statsd.update('blueprint-io-server.storage', len(data)) + element = StoredObject(key, data) + try: + collection.insert(element.__dict__) + except: + return False + return True + + + +def put_blueprint(secret, name, data): + return put(key_for_blueprint(secret, name), data) + + +def put_tarball(secret, name, sha, data): + return put(key_for_tarball(secret, name, sha), data) + + +def url_for(key): + return '{0}://{1}/{2}'.format(protocol, url, key) + + +def url_for_blueprint(secret, name): + return url_for(key_for_blueprint(secret, name)) + + +def url_for_tarball(secret, name, sha): + return url_for(key_for_tarball(secret, name, sha))