Index: configure.ac =================================================================== --- configure.ac (revision 572) +++ configure.ac (working copy) @@ -148,6 +148,7 @@ conduit/dataproviders/FeedModule/Makefile conduit/dataproviders/FlickrModule/Makefile conduit/dataproviders/FlickrModule/FlickrAPI/Makefile +conduit/dataproviders/FotoBilderModule/Makefile conduit/dataproviders/FspotModule/Makefile conduit/dataproviders/GmailModule/Makefile conduit/dataproviders/GmailModule/libgmail-0.1.5/Makefile Index: conduit/datatypes/Photo.py =================================================================== --- conduit/datatypes/Photo.py (revision 0) +++ conduit/datatypes/Photo.py (revision 0) @@ -0,0 +1,22 @@ +import conduit +from conduit.datatypes import File + +class Photo(File.File): + + ## Tags should be a list of lists, the lower level representing + ## the path to the tag. + + tags = [] + + def __init__(self, URI, tags=None): + File.File.__init__(self, URI) + self.set_tags(tags) + + def get_tags(self): + return self.tags + + def set_tags(self, tags): + self.tags = tags + + def add_tag(self, tag): + self.tags.append(tag) Index: conduit/dataproviders/FotoBilderModule/fotobilderapi/fotobilder.py =================================================================== --- conduit/dataproviders/FotoBilderModule/fotobilderapi/fotobilder.py (revision 0) +++ conduit/dataproviders/FotoBilderModule/fotobilderapi/fotobilder.py (revision 0) @@ -0,0 +1,309 @@ +"""A Python library for accessing FotoBilder, the photo hosting +software that runs pics.livejournal.com + +Stuart Langridge, http://www.kryogenix.org/ +v 0.1, 24/04/2007 +""" + +import os, urlparse +from xml.dom import minidom +from xml.xpath import Evaluate +from md5 import md5 +from ProgressReportingHTTPConnection import ProgressReportingHTTPConnection + +ENDPOINT = "http://pics.livejournal.com/interface/simple" + + +class FotoBilder: + def __init__(self, user, password, progress_callback=None): + self.user = user + self.password = password + self.progress_callback = progress_callback + self.CURRENT_CHALLENGE = "" + scheme,self.server,self.url,urlparams,query,fragment = urlparse.urlparse(ENDPOINT) + self.GetChallenge() # prime the system with a challenge + + def CreateGals(self, galleries): + raise NotImplementedError + + def GetChallenge(self): + """Get a challenge to be used in later authentication. + + You should not need to call this. This library takes care of auth + challenges for you. + + """ + dom = self.__send("GET",{"Mode":"GetChallenge"}) + return Evaluate( + "/FBResponse/GetChallengeResponse/Challenge/text()",dom)[0].nodeValue + + def GetChallenges(self, quantity): + raise NotImplementedError + + def GetGals(self): + dom = self.__send("GET",{"Mode":"GetGals"}) + gals = [] + for galnode in Evaluate("/FBResponse/GetGalsResponse/Gal",dom): + gal = {} + gal.update(dict(galnode.attributes.items())) + for node in galnode.childNodes: + if node.nodeType == dom.TEXT_NODE: continue + if node.nodeName == "GalMembers": + gal["GalMembers"] = [x.getAttribute("id") for x in + Evaluate("GalMember",node)] + elif node.nodeName == "ParentGals": + gal["ParentGals"] = [x.getAttribute("id") for x in + Evaluate("ParentGal",node)] + elif node.nodeName == "ChildGals": + gal["ChildGals"] = [x.getAttribute("id") for x in + Evaluate("ChildGal",node)] + else: + try: + gal[node.nodeName] = node.firstChild.nodeValue + except: + gal[node.nodeName] = '' + gals.append(gal) + return gals + + def GetGalsTree(self): + raise NotImplementedError + + def GetPics(self): + dom = self.__send("GET",{"Mode":"GetPics"}) + pics = [] + for picnode in Evaluate("/FBResponse/GetPicsResponse/Pic",dom): + pic = {} + pic.update(dict(picnode.attributes.items())) + for node in picnode.childNodes: + if node.nodeType == dom.TEXT_NODE: continue + if node.nodeName == "Meta": + try: + pic[node.getAttribute("name")] = node.firstChild.nodeValue + except: + pass + else: + try: + pic[node.nodeName] = node.firstChild.nodeValue + except: + pic[node.nodeName] = '' + pics.append(pic) + return pics + + def GetSecGroups(self): + raise NotImplementedError + + def Login(self): + raise NotImplementedError + + def UploadPic(self, image_filename, galleries=[], + filename_is_receipt=False, filename_is_data=False): + """Upload a picture + + Arguments: + image_filename - a path to an image + galleries - a list of gallery namelists + filename_is_receipt - if you have a receipt for this image, returned + from UploadPrepare, then set image_filename to the receipt and + filename_is_receipt to True. + filename_is_data - if you want to pass imagedata rather than a filename, + set image_filename to the data and filename_is_data to True. + + A gallery namelist is a list of path components for a gallery. So a + gallery "level3" inside a gallery "level2" inside a gallery "level1" + would be passed as ["level1","level2","level3"]. You pass a list + of these lists. Interim galleries will be created. So, if there are + currently no galleries at all, and you pass the following as galleries: + [["level1","level2","gal1"],["level1","level2a","gal2"],["level1a"]] + then at the end there will be galleries /level1/level2/gal1, + /level1/level2a/gal2, and /level1a. + """ + param_data = { + "Mode":"UploadPic" + } + if galleries: + param_data["UploadPic.Gallery._size"] = len(galleries) + count = 0 + for g in galleries: + gname = g[-1] + if len(g) > 1: + pathelements = g[:-1] + pathcount = 0 + param_data["UploadPic.Gallery.%s.Path._size" % count] = len(pathelements) + for pathel in pathelements: + param_data["UploadPic.Gallery.%s.Path.%s" % (count,pathcount)] = pathel + pathcount += 1 + param_data["UploadPic.Gallery.%s.GalName" % count] = gname + count += 1 + if filename_is_receipt: + param_data["UploadPic.Receipt"] = image_filename + return self.__send("PUT",param_data) + elif filename_is_data: + return self.__send("PUT",param_data,image_filename) + else: + fp = open(image_filename,"rb") + image_data = fp.read() + fp.close() + return self.__send("PUT",param_data,image_data) + + def UploadPrepare(self, image_filenames): + """Checks to see if images have been uploaded before. + + Returns a dict keyed on image filename. Each value is a dict with + key "status". If an image has been uploaded already, status == "existing" + and the value dict also contains a key "receipt" with a value that + can be passed to UploadPic. If the image has not already been uploaded + then status == "new". + + Actually calls __UploadPrepare to do the work, because you + can't UploadPrepare more than about 5 images at a time. + + """ + + pics = {} + for i in range(0,len(image_filenames),5): + pics.update(self.__UploadPrepare(image_filenames[i:i+5])) + return pics + + def __UploadPrepare(self,image_filenames): + param_data = {} + param_data["UploadPrepare.Pic._size"] = len(image_filenames) + count = 0 + prepared = {} + for f in image_filenames: + picmagic, picmd5, picsize = self.CalculateImageAttributes(f) + prepared[picmd5] = f + param_data["UploadPrepare.Pic.%s.MD5" % count] = picmd5 + param_data["UploadPrepare.Pic.%s.Magic" % count] = picmagic + param_data["UploadPrepare.Pic.%s.Size" % count] = picsize + count += 1 + + return self.__UploadPrepareSend(param_data,prepared) + + def CalculateImageAttributes(self, image): + fp = open(image,"rb") + data = fp.read() + fp.close() + picmagic = ''.join([('0'+hex(ord(x))[2:])[-2:] for x in data[:10]]) + picmd5 = md5(data).hexdigest() + size = len(data) + return (picmagic, picmd5, size) + + def UploadPrepareWithData(self, image_dict): + """Checks to see if images have been uploaded before. + + Used when you already have magic and md5 and size data, rather + than wanting them calculated for you. + + image_dict - a dictionary of images: + { "path to image" : + { + "md5" : md5 hex digest of image data, + "magic": magic key for image data, + "size": size of image data + }, + ... + } + + """ + pics = {} + items = image_dict.items() + for i in range(0,len(items),5): + pics.update(self.__UploadPrepareWithData(dict(items[i:i+5]))) + print "Sending",i + return pics + + def __UploadPrepareWithData(self, image_dict): + param_data = {} + param_data["UploadPrepare.Pic._size"] = len(image_dict.keys()) + count = 0 + prepared = {} + for filename, data in image_dict.items(): + prepared[data["md5"]] = filename + param_data["UploadPrepare.Pic.%s.MD5" % count] = data["md5"] + param_data["UploadPrepare.Pic.%s.Magic" % count] = data["magic"] + param_data["UploadPrepare.Pic.%s.Size" % count] = data["size"] + count += 1 + return self.__UploadPrepareSend(param_data,prepared) + + def __UploadPrepareSend(self, param_data, prepared): + """Actually send the UploadPrepare data to the server""" + dom = self.__send("GET",param_data) + pics = {} + for picnode in Evaluate('/FBResponse/UploadPrepareResponse/Pic',dom): + picdata = {} + picmd5 = Evaluate("MD5",picnode)[0].firstChild.nodeValue + known = picnode.getAttribute("known") + if known == '': + # some kind of error with this picture + picdata["status"] = "error" + errornodes = Evaluate("Error",picnode) + if errornodes: + picdata["errormessage"] = errornodes[0].firstChild.nodeValue + picdata["errorcode"] = errornodes[0].getAttribute("code") + elif known == '0': + picdata["status"] = "new" + else: + picdata["status"] = "exists" + picdata["receipt"] = Evaluate("Receipt", + picnode)[0].firstChild.nodeValue + picfilename = prepared[picmd5] + pics[picfilename] = picdata + return pics + + def UploadTempFile(self, image_filename): + raise NotImplementedError + + def AbortCurrentConnection(self): + if self.con: self.con.abort_connection() + + def __send(self,method, params, body="", returnxml=True): + self.con = ProgressReportingHTTPConnection(self.server, + progress_callback=self.progress_callback) + headers = self.__params_to_headers(params) + + # explicitly add required headers + headers["X-FB-User"] = self.user + headers["X-FB-GetChallenge"] = "1" + + # if we have a current challenge, use it to add an Auth header + if self.CURRENT_CHALLENGE: + auth = md5(self.CURRENT_CHALLENGE + md5(self.password).hexdigest()).hexdigest() + auth = "crp:%s:%s" % (self.CURRENT_CHALLENGE,auth) + headers["X-FB-Auth"] = auth + self.con.request(method,self.url,body,headers) + try: + response = self.con.getresponse() + except: + raise + data = response.read() + + # Extract the challenge from the response so we've got it for next time + dom = minidom.parseString(data) + try: + self.CURRENT_CHALLENGE = Evaluate( + "/FBResponse/GetChallengeResponse/Challenge/text()",dom)[0].nodeValue + except: + print "FAILFAILFAILFAILFAILFAILFAILFAILFAILFAILFAILFAILFAIL" + print headers + print data + print "FAILFAILFAILFAILFAILFAILFAILFAILFAILFAILFAILFAILFAIL" + raise "Connection to LJ failed" + if returnxml: + return dom + else: + return data + + def __params_to_headers(self,params): + ret = {} + for k,v in params.items(): + ret["X-FB-"+k] = v + return ret + +if __name__ == "__main__": + fb = FotoBilder("stuartlangridge","") + print fb.GetGals() + #print fb.GetPics() + #out = fb.UploadPrepare(["/home/aquarius/Default.png"]) + #receipt = out['/home/aquarius/Default.png']['receipt'] + fb.UploadPic("/home/aquarius/Photos/2002/9/25/LinuxMan.jpg", [["new"]]) + Index: conduit/dataproviders/FotoBilderModule/fotobilderapi/ProgressReportingHTTPConnection.py =================================================================== --- conduit/dataproviders/FotoBilderModule/fotobilderapi/ProgressReportingHTTPConnection.py (revision 0) +++ conduit/dataproviders/FotoBilderModule/fotobilderapi/ProgressReportingHTTPConnection.py (revision 0) @@ -0,0 +1,59 @@ +import httplib, socket + +class ProgressReportingHTTPConnection(httplib.HTTPConnection): + """An HTTPConnection which sends data in small blocks and + calls an optional callback so you can see that it's making progress. + + Pass the callback function as progress_callback to constructor. The callback + function takes two parameters, i and l; i is the amount of data transferred, + l is the total amount of data being sent. + + Also allows you to abort connections. + + """ + def __init__(self, host, port=None, strict=None, progress_callback=None): + httplib.HTTPConnection.__init__(self, host, port, strict) + self.progress_callback = progress_callback + self.ABORT_CONNECTION = False + + def abort_connection(self): + self.ABORT_CONNECTION = True + + def send(self, str): + """Send `str' to the server.""" + if self.sock is None: + if self.auto_open: + try: + self.connect() + except: + self.connect() # try one more time! + else: + raise NotConnected() + + # send the data to the server. if we get a broken pipe, then close + # the socket. we want to reconnect when somebody tries to send again. + # + # NOTE: we DO propagate the error, though, because we cannot simply + # ignore the error... the caller will know if they can retry. + if self.debuglevel > 0: + print "send:", repr(str) + try: + if self.progress_callback: + BLOCK_SIZE = 2048 # 2K blocks + l = len(str) + for i in range(0,l,BLOCK_SIZE): + self.sock.sendall(str[i:i+BLOCK_SIZE]) + self.progress_callback(i,l) + if self.ABORT_CONNECTION: + self.close() + return + + # report completion + self.progress_callback(l,l) + else: + self.sock.sendall(str) + except socket.error, v: + if v[0] == 32: # Broken pipe + self.close() + raise + Index: conduit/dataproviders/FotoBilderModule/FotoBilderModule.py =================================================================== --- conduit/dataproviders/FotoBilderModule/FotoBilderModule.py (revision 0) +++ conduit/dataproviders/FotoBilderModule/FotoBilderModule.py (revision 0) @@ -0,0 +1,128 @@ +""" +FotoBilder Uploader. + +Code shamelessly appropriated from the Flicker Module +""" +import os, sys +import gtk +import traceback +import md5 + +import conduit +from conduit import log,logd,logw +import conduit.Utils as Utils +import conduit.DataProvider as DataProvider +import conduit.Exceptions as Exceptions +import conduit.datatypes.File as File + +Utils.dataprovider_add_dir_to_path(__file__, "fotobilderapi") +from fotobilder import FotoBilder + + +MODULES = { + "FotoBilderSink" : { "type": "dataprovider" } +} + +class FotoBilderSink(DataProvider.DataSink): + + _name_ = "FotoBilder" + _description_ = "Sync Your Live Journal Photos" + _category_ = DataProvider.CATEGORY_WEB + _module_type_ = "sink" + _in_type_ = "file" + _out_type_ = "file" + _icon_ = "image-x-generic" + + ALLOWED_MIMETYPES = ["image/jpeg", "image/png", "image/gif"] + + def __init__(self, *args): + DataProvider.DataSink.__init__(self) + self.need_configuration(True) + + self.username = "" + self.password = "" + self.fapi = None + + def initialize(self): + return True + + def refresh(self): + DataProvider.DataSink.refresh(self) + try: + self.fapi = FotoBilder(self.username, self.password) + except: + logw("Error logging into Live Journal (username %s)\n%s" % + (self.username,traceback.format_exc())) + raise Exceptions.RefreshError + + + def put(self, photo, overwrite, LUID=None): + """ + Accepts a vfs file. Must be made local. + """ + DataProvider.DataSink.put(self, photo, overwrite, LUID) + + originalName = photo.get_filename() + #Gets the local URI (/foo/bar). If this is a remote file then + #it is first transferred to the local filesystem + photoURI = photo.get_local_uri() + + mimeType = photo.get_mimetype() + if mimeType not in FotoBilderSink.ALLOWED_MIMETYPES: + raise Exceptions.SyncronizeError("FotoBilder does not allow uploading %s Files" % mimeType) + + tags = photo.get_tags() + logd("Uploading Photo URI = %s, Mimetype = %s, Original Name = %s, Tags = %s" % + (photoURI, mimeType, originalName, tags)) + # Tags map to galleries + ret = self.fapi.UploadPic(image_filename=photoURI, galleries=tags) + picID = ret.getElementsByTagName("PicID") + logd("Put Photo") + return picID.item(0).nodeValue + + def configure(self, window): + """ + Configure FotoBilder account + """ + def set_username(username): + self.username = username + + def set_password(password): + self.password = password + + items = [ + { + "Name" : "Username:", + "Widget" : gtk.Entry, + "Callback" : set_username, + "InitialValue" : self.username + }, + { + "Name" : "Password:", + "Widget" : gtk.Entry, + "Callback" : set_password, + "InitialValue" : self.password + } + ] + + #We just use a simple configuration dialog + dialog = DataProvider.DataProviderSimpleConfigurator(window, "Configure FotoBilder", items) + #This call blocks + dialog.run() + + if len(self.username) > 0 and len(self.password) > 0: + self.set_configured(True) + + def get_configuration(self): + return { + "username" : self.username, + "password" : self.password, + } + + def set_configuration(self, config): + DataProvider.DataSink.set_configuration(self, config) + if len(self.username) > 0 and len(self.password) > 0: + self.set_configured(True) + + def get_UID(self): + return "%s:%s" % (self.username, self.password) Index: conduit/dataproviders/FotoBilderModule/Makefile.am =================================================================== --- conduit/dataproviders/FotoBilderModule/Makefile.am (revision 0) +++ conduit/dataproviders/FotoBilderModule/Makefile.am (revision 0) @@ -0,0 +1,5 @@ +conduit_handlersdir = $(libdir)/conduit/dataproviders/FotoBilderModule +conduit_handlers_PYTHON = FotoBilderModule.py fotobilder.py ProgressReportingHTTPConnection.py + +clean-local: + rm -rf *.pyc *.pyo Index: conduit/dataproviders/FspotModule/FspotModule.py =================================================================== --- conduit/dataproviders/FspotModule/FspotModule.py (revision 572) +++ conduit/dataproviders/FspotModule/FspotModule.py (working copy) @@ -8,7 +8,7 @@ import conduit.Utils as Utils import conduit.Exceptions import conduit.DataProvider as DataProvider -import conduit.datatypes.File as File +import conduit.datatypes.Photo as Photo MODULES = { "FspotSource" : { "type": "dataprovider" } @@ -65,30 +65,49 @@ con = sqlite.connect(FspotSource.PHOTO_DB) tagCur = con.cursor() photoCur = con.cursor() + photoTagsCur = con.cursor() for tagID in self.enabledTags: tagCur.execute("SELECT photo_id FROM photo_tags WHERE tag_id=%s" % (tagID)) for photoID in tagCur: photoCur.execute("SELECT directory_path, name FROM photos WHERE id=?", (photoID)) - for directory_path, name in photoCur: - #Return the file, loaded from a (local only??) URI - if type(photoID) == tuple: - uid = photoID[0] - else: - logw("Error getting photo ID") - uid = photoID + directory_path, name = photoCur.fetchone() + #Return the file, loaded from a (local only??) URI + if type(photoID) == tuple: + uid = photoID[0] + else: + logw("Error getting photo ID") + uid = photoID - logd("Found photo with name=%s (ID: %s)" % (name,uid)) - self.photos.append( (os.path.join(directory_path, name),uid) ) + tags = [] + photoTagsCur.execute("""SELECT id, name, category_id FROM tags WHERE id IN + (SELECT tag_id FROM photo_tags WHERE photo_id = %s)""" % (photoID)) + for tag_id, tag, category_id in photoTagsCur: + # get parent tags + tagArr = [tag] + while (category_id != 0): + logd("Getting parent tag for %s " % tag) + tagParentCur = con.cursor() + tagParentCur.execute("""SELECT id, name, category_id FROM tags + WHERE id = %s""" % category_id) + tag_id, tag, category_id = tagParentCur.fetchone() + logd("adding parent tag of %s %s %s" % (tag_id, tag, category_id)) + tagArr.insert(0, tag) + logd("current tags are %s" % tagArr) + tags.append(tagArr) + logd("Found photo with name=%s (ID: %s) (tags: %s)" % (name,uid,tags)) + self.photos.append( (os.path.join(directory_path, name),uid,tags) ) + con.close() def get(self, index): DataProvider.DataSource.get(self, index) - photouri, photouid = self.photos[index] + photouri, photouid, tags = self.photos[index] - f = File.File(URI=photouri) + f = Photo.Photo(URI=photouri) f.set_UID(photouid) f.set_open_URI(photouri) + f.set_tags(tags) return f @@ -154,6 +173,23 @@ self.set_configured(True) dlg.destroy() + def set_configuration(self, config): + logd(__name__+" setting configuration") + logd (config) + self.enabledTags = [] + for tag in config.get("tags", []): + self.enabledTags.append(int(tag)) + self.set_configured(True) + + + def get_configuration(self): + logd(__name__+" getting configuration") + logd(self.enabledTags) + strTags = [] + for tag in self.enabledTags: + strTags.append(str(tag)) + return {"tags": strTags} + def get_UID(self): return Utils.get_user_string()