From 4da503a1a8255023bd5778252b09f1943c6d0b82 Mon Sep 17 00:00:00 2001
From: Suvarna <suvarna@rubyeffect.com>
Date: Fri, 18 Mar 2016 19:10:50 +0530
Subject: [PATCH] Proxy authentication while parsing url with Net::HTTP, by
 sending proxy username and password along with proxy details.

---
 lib/cobweb.rb | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/lib/cobweb.rb b/lib/cobweb.rb
index 9b35c80..657d83b 100644
--- a/lib/cobweb.rb
+++ b/lib/cobweb.rb
@@ -67,6 +67,14 @@ def initialize(options = {})
 
   end
 
+  # This method will shift the proxies.
+  def proxy_shift(options = {})
+    @options[:proxy_addr] = options[:proxy_addr]
+    @options[:proxy_port] = options[:proxy_port]
+    @options[:proxy_uname] = options[:proxy_uname]
+    @options[:proxy_pwd] = options[:proxy_pwd]
+  end
+  
   # This method starts the resque based crawl and enqueues the base_url
   def start(base_url)
     raise ":base_url is required" unless base_url
@@ -158,7 +166,12 @@ def get(url, options = @options)
       # retrieve data
       #unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." if @options[:debug]
-        @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
+        if @options[:proxy_uname] && @options[:proxy_pwd]
+          proxy = Net::HTTP::Proxy(@options[:proxy_addr], @options[:proxy_port], @options[:proxy_uname], @options[:proxy_pwd])
+          @http = proxy.start(uri.host, uri.inferred_port)
+        else
+          @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
+        end
       #end
       if uri.scheme == "https"
         @http.use_ssl = true
@@ -337,7 +350,12 @@ def head(url, options = @options)
       # retrieve data
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." unless @options[:quiet]
-        @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
+        if @options[:proxy_uname] && @options[:proxy_pwd]
+          proxy = Net::HTTP::Proxy(@options[:proxy_addr], @options[:proxy_port], @options[:proxy_uname], @options[:proxy_pwd])
+          @http = proxy.start(uri.host, uri.inferred_port)
+        else
+          @http = Net::HTTP.new(uri.host, uri.inferred_port, @options[:proxy_addr], @options[:proxy_port])
+        end
       end
       if uri.scheme == "https"
         @http.use_ssl = true