%21Sargasso/py/http_client.py

"""Async HTTP fetching.

This module is taken from the article
    http://effbot.org/zone/effnews-1.htm
(and the other articles in the series).

Copyright 1995-2005 by Fredrik Lundh.
see http://effbot.org/zone/copyright.htm

With modifications by James Bursa.
"""

import asyncore
import socket, time
import StringIO
import mimetools, urlparse


request_headers = []


class CloseConnection(Exception):
    pass


class async_http(asyncore.dispatcher_with_send):
    # asynchronous http client

    def __init__(self, host, port, path, consumer):
        asyncore.dispatcher_with_send.__init__(self)

        self.host = host
        self.port = port
        self.path = path

        self.consumer = consumer

        self.status = None
        self.header = None

        self.bytes_in = 0
        self.bytes_out = 0

        self.data = ""

        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect((host, port))

    def handle_connect(self):
        # connection succeeded
        headers = ["GET %s HTTP/1.0" % self.path,
                "Host: %s" % self.host] + request_headers
        text = "\r\n".join(headers) + "\r\n\r\n"
        self.send(text)
        self.bytes_out = self.bytes_out + len(text)

    def handle_expt(self):
        # connection failed; notify consumer
        self.close()
        self.consumer.http_failed(self)

    def handle_read(self):

        data = self.recv(2048)
        self.bytes_in = self.bytes_in + len(data)

        if not self.header:
            # check if we've seen a full header
            self.data = self.data + data

            header = self.data.split("\r\n\r\n", 1)
            if len(header) <= 1:
                return
            header, data = header

            # parse header
            fp = StringIO.StringIO(header)
            self.status = fp.readline().split(" ", 2)
            self.header = mimetools.Message(fp)

            self.data = ""

            try:
                self.consumer.http_header(self)
            except CloseConnection:
                self.close()
                return

            if not self.connected:
                return # channel was closed by consumer

        if data:
            self.consumer.receive(data)

    def handle_close(self):
        self.consumer.close()
        self.close()

    def handle_error(self):
        self.consumer.http_failed(self)
        self.close()


def do_request(uri, consumer):

    # turn the uri into a valid request

    scheme, host, path, params, query, fragment = urlparse.urlparse(uri)
    assert scheme == "http", "only supports HTTP requests"
    try:
        host, port = host.split(":", 1)
        port = int(port)
    except (TypeError, ValueError):
        port = 80 # default port

    if not path:
        path = "/"
    if params:
        path = path + ";" + params
    if query:
        path = path + "?" + query

    return async_http(host, port, path, consumer)
1	james	76	"""Async HTTP fetching.
2
3			This module is taken from the article
4			http://effbot.org/zone/effnews-1.htm
5			(and the other articles in the series).
6
7			Copyright 1995-2005 by Fredrik Lundh.
8			see http://effbot.org/zone/copyright.htm
9
10			With modifications by James Bursa.
11			"""
12
13			import asyncore
14			import socket, time
15			import StringIO
16			import mimetools, urlparse
17
18
19			request_headers = []
20
21
22			class CloseConnection(Exception):
23			pass
24
25
26			class async_http(asyncore.dispatcher_with_send):
27			# asynchronous http client
28
29			def __init__(self, host, port, path, consumer):
30			asyncore.dispatcher_with_send.__init__(self)
31
32			self.host = host
33			self.port = port
34			self.path = path
35
36			self.consumer = consumer
37
38			self.status = None
39			self.header = None
40
41			self.bytes_in = 0
42			self.bytes_out = 0
43
44			self.data = ""
45
46			self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
47			self.connect((host, port))
48
49			def handle_connect(self):
50			# connection succeeded
51			headers = ["GET %s HTTP/1.0" % self.path,
52			"Host: %s" % self.host] + request_headers
53			text = "\r\n".join(headers) + "\r\n\r\n"
54			self.send(text)
55			self.bytes_out = self.bytes_out + len(text)
56
57			def handle_expt(self):
58			# connection failed; notify consumer
59			self.close()
60			self.consumer.http_failed(self)
61
62			def handle_read(self):
63
64			data = self.recv(2048)
65			self.bytes_in = self.bytes_in + len(data)
66
67			if not self.header:
68			# check if we've seen a full header
69			self.data = self.data + data
70
71			header = self.data.split("\r\n\r\n", 1)
72			if len(header) <= 1:
73			return
74			header, data = header
75
76			# parse header
77			fp = StringIO.StringIO(header)
78			self.status = fp.readline().split(" ", 2)
79			self.header = mimetools.Message(fp)
80
81			self.data = ""
82
83			try:
84			self.consumer.http_header(self)
85			except CloseConnection:
86			self.close()
87			return
88
89			if not self.connected:
90			return # channel was closed by consumer
91
92			if data:
93			self.consumer.receive(data)
94
95			def handle_close(self):
96			self.consumer.close()
97			self.close()
98
99			def handle_error(self):
100			self.consumer.http_failed(self)
101			self.close()
102
103
104			def do_request(uri, consumer):
105
106			# turn the uri into a valid request
107
108			scheme, host, path, params, query, fragment = urlparse.urlparse(uri)
109			assert scheme == "http", "only supports HTTP requests"
110			try:
111			host, port = host.split(":", 1)
112			port = int(port)
113			except (TypeError, ValueError):
114			port = 80 # default port
115
116			if not path:
117			path = "/"
118			if params:
119			path = path + ";" + params
120			if query:
121			path = path + "?" + query
122
123			return async_http(host, port, path, consumer)