I have been a long time web crawling fan and have often used this library to crawl around the web. +I've found that I always end up repeating the same sort of bootstrapping for some of the super handy +objects that this library supplies. +
+ +Old
+from mechanize import Browser
+br = Browser()
+br.set_handle_equiv(True)br.set_handle_http(True)...from cookielib import LWPCookieJarcookiejar = LWPCookieJar()br.set_cookiejar(cookiejar)br. ....New
+from mechanize import Browser
+br = Browser()
+br.bootstrap()
+
+
+Go crawl the web without worrying about any stupid robots.txt nonsense with your emulated, bootstrapped and user-agent mocking browser
diff --git a/README.txt b/README.txt deleted file mode 100644 index 70a0647..0000000 --- a/README.txt +++ /dev/null @@ -1,7 +0,0 @@ -See INSTALL.txt for installation instructions. - -See docs/html/index.html and docstrings for documentation. - -If you have a git working tree rather than a release, you'll only have -the markdown source, e.g. mechanize/index.txt; release.py is used to -build the HTML docs. diff --git a/mechanize/__init__.py b/mechanize/__init__.py index c4429be..dcdb225 100644 --- a/mechanize/__init__.py +++ b/mechanize/__init__.py @@ -209,3 +209,13 @@ if logger.level is logging.NOTSET: logger.setLevel(logging.CRITICAL) del logger + +import os +try: + import bs4 +except ImportError: + contin = raw_input("This fork of mechanize requires beautifulsoup4 and you don't appear to have it.\nWould you like to proceed? (yes or no): ") + if contin == "yes": + os.system("sudo pip install beautifulsoup4") + else: + pass diff --git a/mechanize/_mechanize.py b/mechanize/_mechanize.py index 5ce71a6..f8d00e1 100644 --- a/mechanize/_mechanize.py +++ b/mechanize/_mechanize.py @@ -667,3 +667,62 @@ def _filter_links(self, links, continue yield link nr = orig_nr + + + + + def set_user_agent(self, name=None): + """ + Author : Wes Madrigal + Email : wesley7879@gmail.com + Github : https://github.com/wesmadrigal + Date : 01-18-14 + + sets a Browser instance user agent + defaults to firefox on a linux machine + can pass kwarg to the name such as: + chrome, ie (internet explorer), firefox + + """ + if not name: + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] + + elif name == 'ie': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)') ] + + elif name == 'chrome': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36') ] + + elif name == 'firefox': + #self.set_user_agent() + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] + + elif name == 'safari': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25') ] + + elif name == 'iphone': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11B511') ] + + else: + print "Error: No such browser %s" % name + print "Defaulting to Firefox" + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] + + + + def bootstrap(self, agent=None): + """ + Author : Wes Madrigal + Email : wesley7879@gmail.com + Github : https://github.com/wesmadrigal + Date : 01-18-14 + A pre-strapped Browser instance with all my personal favorite fixtures + Motivation: + Every time I'm using a mechanize.Browser instance I strap it with these fixtures + Most of the time this is a manual thing, although I've written scripts to do this for me in the past + """ + self.set_handle_equiv(True) + self.set_handle_gzip(True) + self.set_debug_http(True) + self.set_handle_robots(False) + self.set_user_agent()