From 3a390733f14c981fde36dffa16229e5b3f30ae8a Mon Sep 17 00:00:00 2001 From: wes Date: Sat, 18 Jan 2014 18:12:45 -0600 Subject: [PATCH 1/6] Updated the _mechanize.Browser class: added a method called set_user_agent that sets a user agent and defaults to Firefox --- mechanize/_mechanize.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/mechanize/_mechanize.py b/mechanize/_mechanize.py index 5ce71a6..4dbee61 100644 --- a/mechanize/_mechanize.py +++ b/mechanize/_mechanize.py @@ -667,3 +667,38 @@ def _filter_links(self, links, continue yield link nr = orig_nr + + + + """ + Author : Wes Madrigal + Date : 01/18/14 + # sets a Browser instance user agent + # defaults to firefox on a linux machine + # can pass kwarg to the name such as: + # chrome, ie (internet explorer), firefox + + """ + + def set_user_agent(self, name=None): + if not name: + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] + + elif name == 'ie': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)') ] + + elif name == 'chrome': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36') ] + + elif name == 'firefox': + #self.set_user_agent() + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] + + elif name == 'safari': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25') ] + + else: + print "Error: No such browser %s" % name + print "Defaulting to Firefox" + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] + From 402c9816c99a5ca46d65d1e7435564d9931648dd Mon Sep 17 00:00:00 2001 From: wes Date: Sat, 18 Jan 2014 22:58:00 -0600 Subject: [PATCH 2/6] Updated Browser class : added Browser.bootstrap method to bootstrap the instance with fixtures that I commonly use --- mechanize/_mechanize.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/mechanize/_mechanize.py b/mechanize/_mechanize.py index 4dbee61..eed454c 100644 --- a/mechanize/_mechanize.py +++ b/mechanize/_mechanize.py @@ -672,7 +672,9 @@ def _filter_links(self, links, """ Author : Wes Madrigal - Date : 01/18/14 + Email : wesley7879@gmail.com + Github : https://github.com/wesmadrigal + Date : 01-18-14 # sets a Browser instance user agent # defaults to firefox on a linux machine # can pass kwarg to the name such as: @@ -702,3 +704,21 @@ def set_user_agent(self, name=None): print "Defaulting to Firefox" self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] + + """ + Author : Wes Madrigal + Email : wesley7879@gmail.com + Github : https://github.com/wesmadrigal + Date : 01-18-14 + A pre-strapped Browser instance with all my personal favorite fixtures + Motivation: + Every time I'm using a mechanize.Browser instance I strap it with these fixtures + Most of the time this is a manual thing, although I've written scripts to do this for me in the past + """ + + def bootstrap(self, agent=None): + self.set_handle_equiv(True) + self.set_handle_gzip(True) + self.set_debug_http(True) + self.set_handle_robots(False) + self.set_user_agent() From d314f656df9f76139208c326f18e5ccf6ae35d44 Mon Sep 17 00:00:00 2001 From: wes Date: Tue, 21 Jan 2014 22:52:25 -0600 Subject: [PATCH 3/6] Added iPhone option for emulated browser instance --- mechanize/_mechanize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mechanize/_mechanize.py b/mechanize/_mechanize.py index eed454c..4fe7ce9 100644 --- a/mechanize/_mechanize.py +++ b/mechanize/_mechanize.py @@ -698,6 +698,9 @@ def set_user_agent(self, name=None): elif name == 'safari': self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25') ] + + elif name == 'iphone': + self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Mobile/11B511') ] else: print "Error: No such browser %s" % name From 88a205bd0adc2c119310f5a684a6481c91010154 Mon Sep 17 00:00:00 2001 From: madman2890 Date: Tue, 18 Mar 2014 00:30:08 -0500 Subject: [PATCH 4/6] Update and rename README.txt to README.md --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ README.txt | 7 ------- 2 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 README.md delete mode 100644 README.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..3973d55 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +See INSTALL.txt for installation instructions. + +See docs/html/index.html and docstrings for documentation. + +If you have a git working tree rather than a release, you'll only have +the markdown source, e.g. mechanize/index.txt; release.py is used to +build the HTML docs. + + +

Changes

+ +
    +
  • browser bootstrap
    • +
  • cookie jar bootstrap
    • +
+ +

Motivation

+

I have been a long time web crawling fan and have often used this library to crawl around the web. +I've found that I always end up repeating the same sort of bootstrapping for some of the super handy +objects that this library supplies. +

+ +

Example

+

Old

+from mechanize import Browser +
+br = Browser() +
+br.set_handle_equiv(True)
+br.set_handle_http(True)
+...
+from cookielib import LWPCookieJar
+cookiejar = LWPCookieJar()
+br.set_cookiejar(cookiejar)
+br. ....
+
+

New

+from mechanize import Browser +
+br = Browser() +
+br.bootstrap() + + +

Go crawl the web without worrying about any stupid robots.txt nonsense with your emulated, bootstrapped and user-agent mocking browser

diff --git a/README.txt b/README.txt deleted file mode 100644 index 70a0647..0000000 --- a/README.txt +++ /dev/null @@ -1,7 +0,0 @@ -See INSTALL.txt for installation instructions. - -See docs/html/index.html and docstrings for documentation. - -If you have a git working tree rather than a release, you'll only have -the markdown source, e.g. mechanize/index.txt; release.py is used to -build the HTML docs. From 16c7350d0915f98663614cacbbd988ff6910031e Mon Sep 17 00:00:00 2001 From: wesmadrigal Date: Mon, 31 Mar 2014 00:22:28 -0500 Subject: [PATCH 5/6] Adding beautifulsoup4 to the __init__ script as a dependency for extended functionality I'm about to write --- mechanize/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mechanize/__init__.py b/mechanize/__init__.py index c4429be..dcdb225 100644 --- a/mechanize/__init__.py +++ b/mechanize/__init__.py @@ -209,3 +209,13 @@ if logger.level is logging.NOTSET: logger.setLevel(logging.CRITICAL) del logger + +import os +try: + import bs4 +except ImportError: + contin = raw_input("This fork of mechanize requires beautifulsoup4 and you don't appear to have it.\nWould you like to proceed? (yes or no): ") + if contin == "yes": + os.system("sudo pip install beautifulsoup4") + else: + pass From 1832b49387e2b19c84f80ca5c642f4e6d548171e Mon Sep 17 00:00:00 2001 From: wesmadrigal Date: Mon, 31 Mar 2014 23:05:01 -0500 Subject: [PATCH 6/6] moving comments --- mechanize/_mechanize.py | 43 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/mechanize/_mechanize.py b/mechanize/_mechanize.py index 4fe7ce9..f8d00e1 100644 --- a/mechanize/_mechanize.py +++ b/mechanize/_mechanize.py @@ -670,19 +670,20 @@ def _filter_links(self, links, - """ - Author : Wes Madrigal - Email : wesley7879@gmail.com - Github : https://github.com/wesmadrigal - Date : 01-18-14 - # sets a Browser instance user agent - # defaults to firefox on a linux machine - # can pass kwarg to the name such as: - # chrome, ie (internet explorer), firefox - - """ def set_user_agent(self, name=None): + """ + Author : Wes Madrigal + Email : wesley7879@gmail.com + Github : https://github.com/wesmadrigal + Date : 01-18-14 + + sets a Browser instance user agent + defaults to firefox on a linux machine + can pass kwarg to the name such as: + chrome, ie (internet explorer), firefox + + """ if not name: self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] @@ -708,18 +709,18 @@ def set_user_agent(self, name=None): self.addheaders = [ ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9a3pre) Gecko/20070330') ] - """ - Author : Wes Madrigal - Email : wesley7879@gmail.com - Github : https://github.com/wesmadrigal - Date : 01-18-14 - A pre-strapped Browser instance with all my personal favorite fixtures - Motivation: - Every time I'm using a mechanize.Browser instance I strap it with these fixtures - Most of the time this is a manual thing, although I've written scripts to do this for me in the past - """ def bootstrap(self, agent=None): + """ + Author : Wes Madrigal + Email : wesley7879@gmail.com + Github : https://github.com/wesmadrigal + Date : 01-18-14 + A pre-strapped Browser instance with all my personal favorite fixtures + Motivation: + Every time I'm using a mechanize.Browser instance I strap it with these fixtures + Most of the time this is a manual thing, although I've written scripts to do this for me in the past + """ self.set_handle_equiv(True) self.set_handle_gzip(True) self.set_debug_http(True)