%21Sargasso/py/http_client.py

"""Async HTTP fetching.

This module is taken from the article
    http://effbot.org/zone/effnews-1.htm
(and the other articles in the series).

Copyright 1995-2005 by Fredrik Lundh.
see http://effbot.org/zone/copyright.htm

With modifications by James Bursa.
"""

import asyncore
import socket, time
import StringIO
import mimetools, urlparse


request_headers = []


class CloseConnection(Exception):
    pass


class async_http(asyncore.dispatcher_with_send):
    # asynchronous http client

    def __init__(self, host, port, path, consumer):
        asyncore.dispatcher_with_send.__init__(self)

        self.host = host
        self.port = port
        self.path = path

        self.consumer = consumer

        self.status = None
        self.header = None

        self.bytes_in = 0
        self.bytes_out = 0

        self.data = ""

        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect((host, port))

    def handle_connect(self):
        # connection succeeded
        headers = ["GET %s HTTP/1.0" % self.path,
                "Host: %s" % self.host] + request_headers
        text = "\r\n".join(headers) + "\r\n\r\n"
        self.send(text)
        self.bytes_out = self.bytes_out + len(text)

    def handle_expt(self):
        # connection failed; notify consumer
        self.close()
        self.consumer.http_failed(self)

    def handle_read(self):

        data = self.recv(2048)
        self.bytes_in = self.bytes_in + len(data)

        if not self.header:
            # check if we've seen a full header
            self.data = self.data + data

            header = self.data.split("\r\n\r\n", 1)
            if len(header) <= 1:
                return
            header, data = header

            # parse header
            fp = StringIO.StringIO(header)
            self.status = fp.readline().split(" ", 2)
            self.header = mimetools.Message(fp)

            self.data = ""

            try:
                self.consumer.http_header(self)
            except CloseConnection:
                self.close()
                return

            if not self.connected:
                return # channel was closed by consumer

        if data:
            self.consumer.receive(data)

    def handle_close(self):
        self.consumer.close()
        self.close()

    def handle_error(self):
        self.consumer.http_failed(self)
        self.close()


def do_request(uri, consumer):

    # turn the uri into a valid request

    scheme, host, path, params, query, fragment = urlparse.urlparse(uri)
    assert scheme == "http", "only supports HTTP requests"
    try:
        host, port = host.split(":", 1)
        port = int(port)
    except (TypeError, ValueError):
        port = 80 # default port

    if not path:
        path = "/"
    if params:
        path = path + ";" + params
    if query:
        path = path + "?" + query

    return async_http(host, port, path, consumer)
1	"""Async HTTP fetching.
2
3	This module is taken from the article
4	http://effbot.org/zone/effnews-1.htm
5	(and the other articles in the series).
6
7	Copyright 1995-2005 by Fredrik Lundh.
8	see http://effbot.org/zone/copyright.htm
9
10	With modifications by James Bursa.
11	"""
12
13	import asyncore
14	import socket, time
15	import StringIO
16	import mimetools, urlparse
17
18
19	request_headers = []
20
21
22	class CloseConnection(Exception):
23	pass
24
25
26	class async_http(asyncore.dispatcher_with_send):
27	# asynchronous http client
28
29	def __init__(self, host, port, path, consumer):
30	asyncore.dispatcher_with_send.__init__(self)
31
32	self.host = host
33	self.port = port
34	self.path = path
35
36	self.consumer = consumer
37
38	self.status = None
39	self.header = None
40
41	self.bytes_in = 0
42	self.bytes_out = 0
43
44	self.data = ""
45
46	self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
47	self.connect((host, port))
48
49	def handle_connect(self):
50	# connection succeeded
51	headers = ["GET %s HTTP/1.0" % self.path,
52	"Host: %s" % self.host] + request_headers
53	text = "\r\n".join(headers) + "\r\n\r\n"
54	self.send(text)
55	self.bytes_out = self.bytes_out + len(text)
56
57	def handle_expt(self):
58	# connection failed; notify consumer
59	self.close()
60	self.consumer.http_failed(self)
61
62	def handle_read(self):
63
64	data = self.recv(2048)
65	self.bytes_in = self.bytes_in + len(data)
66
67	if not self.header:
68	# check if we've seen a full header
69	self.data = self.data + data
70
71	header = self.data.split("\r\n\r\n", 1)
72	if len(header) <= 1:
73	return
74	header, data = header
75
76	# parse header
77	fp = StringIO.StringIO(header)
78	self.status = fp.readline().split(" ", 2)
79	self.header = mimetools.Message(fp)
80
81	self.data = ""
82
83	try:
84	self.consumer.http_header(self)
85	except CloseConnection:
86	self.close()
87	return
88
89	if not self.connected:
90	return # channel was closed by consumer
91
92	if data:
93	self.consumer.receive(data)
94
95	def handle_close(self):
96	self.consumer.close()
97	self.close()
98
99	def handle_error(self):
100	self.consumer.http_failed(self)
101	self.close()
102
103
104	def do_request(uri, consumer):
105
106	# turn the uri into a valid request
107
108	scheme, host, path, params, query, fragment = urlparse.urlparse(uri)
109	assert scheme == "http", "only supports HTTP requests"
110	try:
111	host, port = host.split(":", 1)
112	port = int(port)
113	except (TypeError, ValueError):
114	port = 80 # default port
115
116	if not path:
117	path = "/"
118	if params:
119	path = path + ";" + params
120	if query:
121	path = path + "?" + query
122
123	return async_http(host, port, path, consumer)