Profile Report Profile Time on Sat Jun 20 06:59:14 2009
Reported Time on Tue Jun 23 02:22:32 2009

Total Time : 12.5950 CPU second
Total Func Call: 1285 function calls

Top 20 functions data
ncalls tottime percall cumtime percall filename:lineno(function)
9 7.4059 0.8229 7.4060 0.8229 /usr/lib/python2.6/socket.py : 194 (accept)
13 5.0035 0.3849 5.0041 0.3849 /usr/lib/python2.6/threading.py : 228 (wait)
1 0.1006 0.1006 0.1155 0.1155 /var/lib/python-support/python2.6/web/wsgiserver/__init__.py : 1283 (start)
1 0.0254 0.0254 0.0316 0.0316 /usr/lib/python2.6/mimetypes.py : 199 (readfp)
23 0.0090 0.0004 0.0092 0.0004 /usr/lib/python2.6/threading.py : 179 (__init__)
610 0.0079 0.0000 0.0079 0.0000 /usr/lib/python2.6/mimetypes.py : 71 (add_type)
1 0.0061 0.0061 0.0068 0.0068 /var/lib/python-support/python2.6/OpenSSL/tsafe.py : 9 (Connection)
1 0.0058 0.0058 0.0060 0.0060 /var/lib/python-support/python2.6/web/wsgiserver/__init__.py : 1366 (SSLConnection)
1 0.0037 0.0037 0.0238 0.0238 /var/lib/python-support/python2.6/web/wsgiserver/__init__.py : 77 (<module>)
1 0.0032 0.0032 0.0102 0.0102 /var/lib/python-support/python2.6/OpenSSL/__init__.py : 10 (<module>)
1 0.0025 0.0025 12.5948 12.5948 /var/lib/python-support/python2.6/web/httpserver.py : 130 (runsimple)
1 0.0025 0.0025 0.0404 0.0404 /usr/lib/python2.6/SimpleHTTPServer.py : 6 (<module>)
77 0.0013 0.0000 0.0013 0.0000 <string> : 1 (<module>)
10 0.0011 0.0001 0.0039 0.0004 /usr/lib/python2.6/threading.py : 461 (start)
1 0.0010 0.0010 0.0017 0.0017 /usr/lib/python2.6/BaseHTTPServer.py : 18 (<module>)
1 0.0010 0.0010 0.0027 0.0027 /usr/lib/python2.6/mimetypes.py : 57 (__init__)
1 0.0009 0.0009 0.0018 0.0018 /usr/lib/python2.6/Queue.py : 1 (<module>)
1 0.0008 0.0008 0.0008 0.0008 /usr/lib/python2.6/heapq.py : 31 (<module>)
1 0.0008 0.0008 0.0008 0.0008 /usr/lib/python2.6/shutil.py : 5 (<module>)
10 0.0006 0.0001 0.0106 0.0011 /usr/lib/python2.6/threading.py : 424 (__init__)
/usr/lib/python2.6/BaseHTTPServer.py
     time      num code
1
"""HTTP server base class.
2
3
Note: the class in this module doesn't implement any HTTP request; see
4
SimpleHTTPServer for simple implementations of GET, HEAD and POST
5
(including CGI scripts).  It does, however, optionally implement HTTP/1.1
6
persistent connections, as of version 0.3.
7
8
Contents:
9
10
- BaseHTTPRequestHandler: HTTP request handler base class
11
- test: test function
12
13
XXX To do:
14
15
- log requests even later (to capture byte count)
16
- log user-agent header and other interesting goodies
17
- send error log to separate file
0.000895 sec 18
"""
19
20
21
# See also:
22
#
23
# HTTP Working Group                                        T. Berners-Lee
24
# INTERNET-DRAFT                                            R. T. Fielding
25
# <draft-ietf-http-v10-spec-00.txt>                     H. Frystyk Nielsen
26
# Expires September 8, 1995                                  March 8, 1995
27
#
28
# URL: http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt
29
#
30
# and
31
#
32
# Network Working Group                                      R. Fielding
33
# Request for Comments: 2616                                       et al
34
# Obsoletes: 2068                                              June 1999
35
# Category: Standards Track
36
#
37
# URL: http://www.faqs.org/rfcs/rfc2616.html
38
39
# Log files
40
# ---------
41
#
42
# Here's a quote from the NCSA httpd docs about log file format.
43
#
44
# | The logfile format is as follows. Each line consists of:
45
# |
46
# | host rfc931 authuser [DD/Mon/YYYY:hh:mm:ss] "request" ddd bbbb
47
# |
48
# |        host: Either the DNS name or the IP number of the remote client
49
# |        rfc931: Any information returned by identd for this person,
50
# |                - otherwise.
51
# |        authuser: If user sent a userid for authentication, the user name,
52
# |                  - otherwise.
53
# |        DD: Day
54
# |        Mon: Month (calendar name)
55
# |        YYYY: Year
56
# |        hh: hour (24-hour format, the machine's timezone)
57
# |        mm: minutes
58
# |        ss: seconds
59
# |        request: The first line of the HTTP request as sent by the client.
60
# |        ddd: the status code returned by the server, - if not available.
61
# |        bbbb: the total number of bytes sent,
62
# |              *not including the HTTP/1.0 header*, - if not available
63
# |
64
# | You can determine the name of the file accessed through request.
65
#
66
# (Actually, the latter is only true if you know the server configuration
67
# at the time the request was made!)
68
2e-06 sec 69
__version__ = "0.3"
70
3e-06 sec 71
__all__ = ["HTTPServer", "BaseHTTPRequestHandler"]
72
3e-06 sec 73
import sys
9e-06 sec 74
import time
5e-06 sec 75
import socket # For gethostbyaddr()
5e-06 sec 76
from warnings import filterwarnings, catch_warnings
1.3e-05 sec 77
with catch_warnings():
3e-06 sec 78
    if sys.py3kwarning:
79
        filterwarnings("ignore", ".*mimetools has been removed",
80
                        DeprecationWarning)
3e-06 sec 81
    import mimetools
6e-06 sec 82
import SocketServer
83
84
# Default error message template
85
DEFAULT_ERROR_MESSAGE = """\
86
<head>
87
<title>Error response</title>
88
</head>
89
<body>
90
<h1>Error response</h1>
91
<p>Error code %(code)d.
92
<p>Message: %(message)s.
93
<p>Error code explanation: %(code)s = %(explain)s.
94
</body>
2.1e-05 sec 95
"""
96
3e-06 sec 97
DEFAULT_ERROR_CONTENT_TYPE = "text/html"
98
2e-06 sec 99
def _quote_html(html):
100
    return html.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
101
1e-05 sec 102
class HTTPServer(SocketServer.TCPServer):
103
3e-06 sec 104
    allow_reuse_address = 1    # Seems to make sense in testing environment
105
1e-06 sec 106
    def server_bind(self):
107
        """Override server_bind to store the server name."""
108
        SocketServer.TCPServer.server_bind(self)
109
        host, port = self.socket.getsockname()[:2]
110
        self.server_name = socket.getfqdn(host)
111
        self.server_port = port
112
113
1.1e-05 sec 114
class BaseHTTPRequestHandler(SocketServer.StreamRequestHandler):
115
116
    """HTTP request handler base class.
117
118
    The following explanation of HTTP serves to guide you through the
119
    code as well as to expose any misunderstandings I may have about
120
    HTTP (so you don't need to read the code to figure out I'm wrong
121
    :-).
122
123
    HTTP (HyperText Transfer Protocol) is an extensible protocol on
124
    top of a reliable stream transport (e.g. TCP/IP).  The protocol
125
    recognizes three parts to a request:
126
127
    1. One line identifying the request type and path
128
    2. An optional set of RFC-822-style headers
129
    3. An optional data part
130
131
    The headers and data are separated by a blank line.
132
133
    The first line of the request has the form
134
135
    <command> <path> <version>
136
137
    where <command> is a (case-sensitive) keyword such as GET or POST,
138
    <path> is a string containing path information for the request,
139
    and <version> should be the string "HTTP/1.0" or "HTTP/1.1".
140
    <path> is encoded using the URL encoding scheme (using %xx to signify
141
    the ASCII character with hex code xx).
142
143
    The specification specifies that lines are separated by CRLF but
144
    for compatibility with the widest range of clients recommends
145
    servers also handle LF.  Similarly, whitespace in the request line
146
    is treated sensibly (allowing multiple spaces between components
147
    and allowing trailing whitespace).
148
149
    Similarly, for output, lines ought to be separated by CRLF pairs
150
    but most clients grok LF characters just fine.
151
152
    If the first line of the request has the form
153
154
    <command> <path>
155
156
    (i.e. <version> is left out) then this is assumed to be an HTTP
157
    0.9 request; this form has no optional headers and data part and
158
    the reply consists of just the data.
159
160
    The reply form of the HTTP 1.x protocol again has three parts:
161
162
    1. One line giving the response code
163
    2. An optional set of RFC-822-style headers
164
    3. The data
165
166
    Again, the headers and data are separated by a blank line.
167
168
    The response code line has the form
169
170
    <version> <responsecode> <responsestring>
171
172
    where <version> is the protocol version ("HTTP/1.0" or "HTTP/1.1"),
173
    <responsecode> is a 3-digit response code indicating success or
174
    failure of the request, and <responsestring> is an optional
175
    human-readable string explaining what the response code means.
176
177
    This server parses the request and the headers, and then calls a
178
    function specific to the request type (<command>).  Specifically,
179
    a request SPAM will be handled by a method do_SPAM().  If no
180
    such method exists the server sends an error response to the
181
    client.  If it exists, it is called with no arguments:
182
183
    do_SPAM()
184
185
    Note that the request name is case sensitive (i.e. SPAM and spam
186
    are different requests).
187
188
    The various request details are stored in instance variables:
189
190
    - client_address is the client IP address in the form (host,
191
    port);
192
193
    - command, path and version are the broken-down request line;
194
195
    - headers is an instance of mimetools.Message (or a derived
196
    class) containing the header information;
197
198
    - rfile is a file object open for reading positioned at the
199
    start of the optional input data part;
200
201
    - wfile is a file object open for writing.
202
203
    IT IS IMPORTANT TO ADHERE TO THE PROTOCOL FOR WRITING!
204
205
    The first thing to be written must be the response line.  Then
206
    follow 0 or more header lines, then a blank line, and then the
207
    actual data (if any).  The meaning of the header lines depends on
208
    the command executed by the server; in most cases, when data is
209
    returned, there should be at least one header line of the form
210
211
    Content-type: <type>/<subtype>
212
213
    where <type> and <subtype> should be registered MIME types,
214
    e.g. "text/html" or "text/plain".
215
2e-06 sec 216
    """
217
218
    # The Python system version, truncated to its first component.
2e-06 sec 219
    sys_version = "Python/" + sys.version.split()[0]
220
221
    # The server software version.  You may want to override this.
222
    # The format is multiple whitespace-separated strings,
223
    # where each string is of the form name[/version].
1.5e-05 sec 224
    server_version = "BaseHTTP/" + __version__
225
226
    # The default request version.  This only affects responses up until
227
    # the point where the request line is parsed, so it mainly decides what
228
    # the client gets back when sending a malformed request line.
229
    # Most web servers default to HTTP 0.9, i.e. don't send a status line.
2e-06 sec 230
    default_request_version = "HTTP/0.9"
231
3e-06 sec 232
    def parse_request(self):
233
        """Parse a request (internal).
234
235
        The request should be stored in self.raw_requestline; the results
236
        are in self.command, self.path, self.request_version and
237
        self.headers.
238
239
        Return True for success, False for failure; on failure, an
240
        error is sent back.
241
242
        """
243
        self.command = None  # set in case of error on the first line
244
        self.request_version = version = self.default_request_version
245
        self.close_connection = 1
246
        requestline = self.raw_requestline
247
        if requestline[-2:] == '\r\n':
248
            requestline = requestline[:-2]
249
        elif requestline[-1:] == '\n':
250
            requestline = requestline[:-1]
251
        self.requestline = requestline
252
        words = requestline.split()
253
        if len(words) == 3:
254
            [command, path, version] = words
255
            if version[:5] != 'HTTP/':
256
                self.send_error(400, "Bad request version (%r)" % version)
257
                return False
258
            try:
259
                base_version_number = version.split('/', 1)[1]
260
                version_number = base_version_number.split(".")
261
                # RFC 2145 section 3.1 says there can be only one "." and
262
                #   - major and minor numbers MUST be treated as
263
                #      separate integers;
264
                #   - HTTP/2.4 is a lower version than HTTP/2.13, which in
265
                #      turn is lower than HTTP/12.3;
266
                #   - Leading zeros MUST be ignored by recipients.
267
                if len(version_number) != 2:
268
                    raise ValueError
269
                version_number = int(version_number[0]), int(version_number[1])
270
            except (ValueError, IndexError):
271
                self.send_error(400, "Bad request version (%r)" % version)
272
                return False
273
            if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1":
274
                self.close_connection = 0
275
            if version_number >= (2, 0):
276
                self.send_error(505,
277
                          "Invalid HTTP Version (%s)" % base_version_number)
278
                return False
279
        elif len(words) == 2:
280
            [command, path] = words
281
            self.close_connection = 1
282
            if command != 'GET':
283
                self.send_error(400,
284
                                "Bad HTTP/0.9 request type (%r)" % command)
285
                return False
286
        elif not words:
287
            return False
288
        else:
289
            self.send_error(400, "Bad request syntax (%r)" % requestline)
290
            return False
291
        self.command, self.path, self.request_version = command, path, version
292
293
        # Examine the headers and look for a Connection directive
294
        self.headers = self.MessageClass(self.rfile, 0)
295
296
        conntype = self.headers.get('Connection', "")
297
        if conntype.lower() == 'close':
298
            self.close_connection = 1
299
        elif (conntype.lower() == 'keep-alive' and
300
              self.protocol_version >= "HTTP/1.1"):
301
            self.close_connection = 0
302
        return True
303
3e-06 sec 304
    def handle_one_request(self):
305
        """Handle a single HTTP request.
306
307
        You normally don't need to override this method; see the class
308
        __doc__ string for information on how to handle specific HTTP
309
        commands such as GET and POST.
310
311
        """
312
        self.raw_requestline = self.rfile.readline()
313
        if not self.raw_requestline:
314
            self.close_connection = 1
315
            return
316
        if not self.parse_request(): # An error code has been sent, just exit
317
            return
318
        mname = 'do_' + self.command
319
        if not hasattr(self, mname):
320
            self.send_error(501, "Unsupported method (%r)" % self.command)
321
            return
322
        method = getattr(self, mname)
323
        method()
324
3e-06 sec 325
    def handle(self):
326
        """Handle multiple requests if necessary."""
327
        self.close_connection = 1
328
329
        self.handle_one_request()
330
        while not self.close_connection:
331
            self.handle_one_request()
332
2e-06 sec 333
    def send_error(self, code, message=None):
334
        """Send and log an error reply.
335
336
        Arguments are the error code, and a detailed message.
337
        The detailed message defaults to the short entry matching the
338
        response code.
339
340
        This sends an error response (so it must be called before any
341
        output has been generated), logs the error, and finally sends
342
        a piece of HTML explaining the error to the user.
343
344
        """
345
346
        try:
347
            short, long = self.responses[code]
348
        except KeyError:
349
            short, long = '???', '???'
350
        if message is None:
351
            message = short
352
        explain = long
353
        self.log_error("code %d, message %s", code, message)
354
        # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201)
355
        content = (self.error_message_format %
356
                   {'code': code, 'message': _quote_html(message), 'explain': explain})
357
        self.send_response(code, message)
358
        self.send_header("Content-Type", self.error_content_type)
359
        self.send_header('Connection', 'close')
360
        self.end_headers()
361
        if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
362
            self.wfile.write(content)
363
3e-06 sec 364
    error_message_format = DEFAULT_ERROR_MESSAGE
2e-06 sec 365
    error_content_type = DEFAULT_ERROR_CONTENT_TYPE
366
3e-06 sec 367
    def send_response(self, code, message=None):
368
        """Send the response header and log the response code.
369
370
        Also send two standard headers with the server software
371
        version and the current date.
372
373
        """
374
        self.log_request(code)
375
        if message is None:
376
            if code in self.responses:
377
                message = self.responses[code][0]
378
            else:
379
                message = ''
380
        if self.request_version != 'HTTP/0.9':
381
            self.wfile.write("%s %d %s\r\n" %
382
                             (self.protocol_version, code, message))
383
            # print (self.protocol_version, code, message)
384
        self.send_header('Server', self.version_string())
385
        self.send_header('Date', self.date_time_string())
386
2e-06 sec 387
    def send_header(self, keyword, value):
388
        """Send a MIME header."""
389
        if self.request_version != 'HTTP/0.9':
390
            self.wfile.write("%s: %s\r\n" % (keyword, value))
391
392
        if keyword.lower() == 'connection':
393
            if value.lower() == 'close':
394
                self.close_connection = 1
395
            elif value.lower() == 'keep-alive':
396
                self.close_connection = 0
397
2e-06 sec 398
    def end_headers(self):
399
        """Send the blank line ending the MIME headers."""
400
        if self.request_version != 'HTTP/0.9':
401
            self.wfile.write("\r\n")
402
3e-06 sec 403
    def log_request(self, code='-', size='-'):
404
        """Log an accepted request.
405
406
        This is called by send_response().
407
408
        """
409
410
        self.log_message('"%s" %s %s',
411
                         self.requestline, str(code), str(size))
412
3e-06 sec 413
    def log_error(self, format, *args):
414
        """Log an error.
415
416
        This is called when a request cannot be fulfilled.  By
417
        default it passes the message on to log_message().
418
419
        Arguments are the same as for log_message().
420
421
        XXX This should go to the separate error log.
422
423
        """
424
425
        self.log_message(format, *args)
426
2e-06 sec 427
    def log_message(self, format, *args):
428
        """Log an arbitrary message.
429
430
        This is used by all other logging functions.  Override
431
        it if you have specific logging wishes.
432
433
        The first argument, FORMAT, is a format string for the
434
        message to be logged.  If the format string contains
435
        any % escapes requiring parameters, they should be
436
        specified as subsequent arguments (it's just like
437
        printf!).
438
439
        The client host and current date/time are prefixed to
440
        every message.
441
442
        """
443
444
        sys.stderr.write("%s - - [%s] %s\n" %
445
                         (self.address_string(),
446
                          self.log_date_time_string(),
447
                          format%args))
448
2e-06 sec 449
    def version_string(self):
450
        """Return the server software version string."""
451
        return self.server_version + ' ' + self.sys_version
452
3e-06 sec 453
    def date_time_string(self, timestamp=None):
454
        """Return the current date and time formatted for a message header."""
455
        if timestamp is None:
456
            timestamp = time.time()
457
        year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp)
458
        s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
459
                self.weekdayname[wd],
460
                day, self.monthname[month], year,
461
                hh, mm, ss)
462
        return s
463
2e-06 sec 464
    def log_date_time_string(self):
465
        """Return the current time formatted for logging."""
466
        now = time.time()
467
        year, month, day, hh, mm, ss, x, y, z = time.localtime(now)
468
        s = "%02d/%3s/%04d %02d:%02d:%02d" % (
469
                day, self.monthname[month], year, hh, mm, ss)
470
        return s
471
3e-06 sec 472
    weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
473
3e-06 sec 474
    monthname = [None,
2e-06 sec 475
                 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
2e-06 sec 476
                 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
477
6e-06 sec 478
    def address_string(self):
479
        """Return the client address formatted for logging.
480
481
        This version looks up the full hostname using gethostbyaddr(),
482
        and tries to find a name that contains at least one dot.
483
484
        """
485
486
        host, port = self.client_address[:2]
487
        return socket.getfqdn(host)
488
489
    # Essentially static class variables
490
491
    # The version of the HTTP protocol we support.
492
    # Set this to HTTP/1.1 to enable automatic keepalive
3e-06 sec 493
    protocol_version = "HTTP/1.0"
494
495
    # The Message-like class used to parse headers
3e-06 sec 496
    MessageClass = mimetools.Message
497
498
    # Table mapping response codes to messages; entries have the
499
    # form {code: (shortmessage, longmessage)}.
500
    # See RFC 2616.
4e-06 sec 501
    responses = {
3e-06 sec 502
        100: ('Continue', 'Request received, please continue'),
503
        101: ('Switching Protocols',
4e-06 sec 504
              'Switching to new protocol; obey Upgrade header'),
505
2e-06 sec 506
        200: ('OK', 'Request fulfilled, document follows'),
2e-06 sec 507
        201: ('Created', 'Document created, URL follows'),
508
        202: ('Accepted',
2e-06 sec 509
              'Request accepted, processing continues off-line'),
2e-06 sec 510
        203: ('Non-Authoritative Information', 'Request fulfilled from cache'),
2e-06 sec 511
        204: ('No Content', 'Request fulfilled, nothing follows'),
2e-06 sec 512
        205: ('Reset Content', 'Clear input form for further input.'),
2e-06 sec 513
        206: ('Partial Content', 'Partial content follows.'),
514
515
        300: ('Multiple Choices',
2e-06 sec 516
              'Object has several resources -- see URI list'),
2e-06 sec 517
        301: ('Moved Permanently', 'Object moved permanently -- see URI list'),
2e-06 sec 518
        302: ('Found', 'Object moved temporarily -- see URI list'),
3e-06 sec 519
        303: ('See Other', 'Object moved -- see Method and URL list'),
520
        304: ('Not Modified',
2e-06 sec 521
              'Document has not changed since given time'),
522
        305: ('Use Proxy',
2e-06 sec 523
              'You must use proxy specified in Location to access this '
524
              'resource.'),
525
        307: ('Temporary Redirect',
2e-06 sec 526
              'Object moved temporarily -- see URI list'),
527
528
        400: ('Bad Request',
2e-06 sec 529
              'Bad request syntax or unsupported method'),
530
        401: ('Unauthorized',
2e-06 sec 531
              'No permission -- see authorization schemes'),
532
        402: ('Payment Required',
3e-06 sec 533
              'No payment -- see charging schemes'),
534
        403: ('Forbidden',
1e-06 sec 535
              'Request forbidden -- authorization will not help'),
3e-06 sec 536
        404: ('Not Found', 'Nothing matches the given URI'),
537
        405: ('Method Not Allowed',
2e-06 sec 538
              'Specified method is invalid for this server.'),
2e-06 sec 539
        406: ('Not Acceptable', 'URI not available in preferred format.'),
3e-06 sec 540
        407: ('Proxy Authentication Required', 'You must authenticate with '
541
              'this proxy before proceeding.'),
3e-06 sec 542
        408: ('Request Timeout', 'Request timed out; try again later.'),
2e-06 sec 543
        409: ('Conflict', 'Request conflict.'),
544
        410: ('Gone',
2e-06 sec 545
              'URI no longer exists and has been permanently removed.'),
3e-06 sec 546
        411: ('Length Required', 'Client must specify Content-Length.'),
2e-06 sec 547
        412: ('Precondition Failed', 'Precondition in headers is false.'),
2e-06 sec 548
        413: ('Request Entity Too Large', 'Entity is too large.'),
2e-06 sec 549
        414: ('Request-URI Too Long', 'URI is too long.'),
3e-06 sec 550
        415: ('Unsupported Media Type', 'Entity body in unsupported format.'),
551
        416: ('Requested Range Not Satisfiable',
2e-06 sec 552
              'Cannot satisfy request range.'),
553
        417: ('Expectation Failed',
2e-06 sec 554
              'Expect condition could not be satisfied.'),
555
3e-06 sec 556
        500: ('Internal Server Error', 'Server got itself in trouble'),
557
        501: ('Not Implemented',
2e-06 sec 558
              'Server does not support this operation'),
2e-06 sec 559
        502: ('Bad Gateway', 'Invalid responses from another server/proxy.'),
560
        503: ('Service Unavailable',
3e-06 sec 561
              'The server cannot process the request due to a high load'),
562
        504: ('Gateway Timeout',
2e-06 sec 563
              'The gateway server did not receive a timely response'),
2e-06 sec 564
        505: ('HTTP Version Not Supported', 'Cannot fulfill request.'),
565
        }
566
567
9e-06 sec 568
def test(HandlerClass = BaseHTTPRequestHandler,
2e-06 sec 569
         ServerClass = HTTPServer, protocol="HTTP/1.0"):
570
    """Test the HTTP request handler class.
571
572
    This runs an HTTP server on port 8000 (or the first command line
573
    argument).
574
575
    """
576
577
    if sys.argv[1:]:
578
        port = int(sys.argv[1])
579
    else:
580
        port = 8000
581
    server_address = ('', port)
582
583
    HandlerClass.protocol_version = protocol
584
    httpd = ServerClass(server_address, HandlerClass)
585
586
    sa = httpd.socket.getsockname()
587
    print "Serving HTTP on", sa[0], "port", sa[1], "..."
588
    httpd.serve_forever()
589
590
3e-06 sec 591
if __name__ == '__main__':
592
    test()
/usr/lib/python2.6/Queue.py
     time      num code
0.000698 sec 1
"""A multi-producer, multi-consumer queue."""
2
2e-06 sec 3
from time import time as _time
1.9e-05 sec 4
from collections import deque
1e-05 sec 5
import heapq
6
2.5e-05 sec 7
__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue']
8
1.3e-05 sec 9
class Empty(Exception):
2e-06 sec 10
    "Exception raised by Queue.get(block=0)/get_nowait()."
2e-06 sec 11
    pass
12
0.00012 sec 13
class Full(Exception):
2e-06 sec 14
    "Exception raised by Queue.put(block=0)/put_nowait()."
2e-06 sec 15
    pass
16
7.4e-05 sec 17
class Queue:
18
    """Create a queue object with a given maximum size.
19
20
    If maxsize is <= 0, the queue size is infinite.
1e-06 sec 21
    """
1.3e-05 sec 22
    def __init__(self, maxsize=0):
1e-06 sec 23
        try:
2e-06 sec 24
            import threading
25
        except ImportError:
26
            import dummy_threading as threading
8e-06 sec 27
        self.maxsize = maxsize
3e-06 sec 28
        self._init(maxsize)
29
        # mutex must be held whenever the queue is mutating.  All methods
30
        # that acquire mutex must release it before returning.  mutex
31
        # is shared between the three conditions, so acquiring and
32
        # releasing the conditions also acquires and releases mutex.
2e-06 sec 33
        self.mutex = threading.Lock()
34
        # Notify not_empty whenever an item is added to the queue; a
35
        # thread waiting to get is notified then.
6e-06 sec 36
        self.not_empty = threading.Condition(self.mutex)
37
        # Notify not_full whenever an item is removed from the queue;
38
        # a thread waiting to put is notified then.
3e-06 sec 39
        self.not_full = threading.Condition(self.mutex)
40
        # Notify all_tasks_done whenever the number of unfinished tasks
41
        # drops to zero; thread waiting to join() is notified to resume
2e-06 sec 42
        self.all_tasks_done = threading.Condition(self.mutex)
4e-06 sec 43
        self.unfinished_tasks = 0
44
3e-06 sec 45
    def task_done(self):
46
        """Indicate that a formerly enqueued task is complete.
47
48
        Used by Queue consumer threads.  For each get() used to fetch a task,
49
        a subsequent call to task_done() tells the queue that the processing
50
        on the task is complete.
51
52
        If a join() is currently blocking, it will resume when all items
53
        have been processed (meaning that a task_done() call was received
54
        for every item that had been put() into the queue).
55
56
        Raises a ValueError if called more times than there were items
57
        placed in the queue.
58
        """
59
        self.all_tasks_done.acquire()
60
        try:
61
            unfinished = self.unfinished_tasks - 1
62
            if unfinished <= 0:
63
                if unfinished < 0:
64
                    raise ValueError('task_done() called too many times')
65
                self.all_tasks_done.notify_all()
66
            self.unfinished_tasks = unfinished
67
        finally:
68
            self.all_tasks_done.release()
69
2e-06 sec 70
    def join(self):
71
        """Blocks until all items in the Queue have been gotten and processed.
72
73
        The count of unfinished tasks goes up whenever an item is added to the
74
        queue. The count goes down whenever a consumer thread calls task_done()
75
        to indicate the item was retrieved and all work on it is complete.
76
77
        When the count of unfinished tasks drops to zero, join() unblocks.
78
        """
79
        self.all_tasks_done.acquire()
80
        try:
81
            while self.unfinished_tasks:
82
                self.all_tasks_done.wait()
83
        finally:
84
            self.all_tasks_done.release()
85
2e-06 sec 86
    def qsize(self):
87
        """Return the approximate size of the queue (not reliable!)."""
88
        self.mutex.acquire()
89
        n = self._qsize()
90
        self.mutex.release()
91
        return n
92
4e-06 sec 93
    def empty(self):
94
        """Return True if the queue is empty, False otherwise (not reliable!)."""
95
        self.mutex.acquire()
96
        n = not self._qsize()
97
        self.mutex.release()
98
        return n
99
3e-06 sec 100
    def full(self):
101
        """Return True if the queue is full, False otherwise (not reliable!)."""
102
        self.mutex.acquire()
103
        n = 0 < self.maxsize == self._qsize()
104
        self.mutex.release()
105
        return n
106
8.6e-05 sec 107
    def put(self, item, block=True, timeout=None):
108
        """Put an item into the queue.
109
110
        If optional args 'block' is true and 'timeout' is None (the default),
111
        block if necessary until a free slot is available. If 'timeout' is
112
        a positive number, it blocks at most 'timeout' seconds and raises
113
        the Full exception if no free slot was available within that time.
114
        Otherwise ('block' is false), put an item on the queue if a free slot
115
        is immediately available, else raise the Full exception ('timeout'
116
        is ignored in that case).
117
        """
1.8e-05 sec 118
        self.not_full.acquire()
4.5e-05 sec 119
        try:
2e-05 sec 120
            if self.maxsize > 0:
121
                if not block:
122
                    if self._qsize() == self.maxsize:
123
                        raise Full
124
                elif timeout is None:
125
                    while self._qsize() == self.maxsize:
126
                        self.not_full.wait()
127
                elif timeout < 0:
128
                    raise ValueError("'timeout' must be a positive number")
129
                else:
130
                    endtime = _time() + timeout
131
                    while self._qsize() == self.maxsize:
132
                        remaining = endtime - _time()
133
                        if remaining <= 0.0:
134
                            raise Full
135
                        self.not_full.wait(remaining)
3e-05 sec 136
            self._put(item)
2.2e-05 sec 137
            self.unfinished_tasks += 1
3.5e-05 sec 138
            self.not_empty.notify()
139
        finally:
2.7e-05 sec 140
            self.not_full.release()
141
3e-06 sec 142
    def put_nowait(self, item):
143
        """Put an item into the queue without blocking.
144
145
        Only enqueue the item if a free slot is immediately available.
146
        Otherwise raise the Full exception.
147
        """
148
        return self.put(item, False)
149
2e-06 sec 150
    def get(self, block=True, timeout=None):
151
        """Remove and return an item from the queue.
152
153
        If optional args 'block' is true and 'timeout' is None (the default),
154
        block if necessary until an item is available. If 'timeout' is
155
        a positive number, it blocks at most 'timeout' seconds and raises
156
        the Empty exception if no item was available within that time.
157
        Otherwise ('block' is false), return an item if one is immediately
158
        available, else raise the Empty exception ('timeout' is ignored
159
        in that case).
160
        """
161
        self.not_empty.acquire()
162
        try:
163
            if not block:
164
                if not self._qsize():
165
                    raise Empty
166
            elif timeout is None:
167
                while not self._qsize():
168
                    self.not_empty.wait()
169
            elif timeout < 0:
170
                raise ValueError("'timeout' must be a positive number")
171
            else:
172
                endtime = _time() + timeout
173
                while not self._qsize():
174
                    remaining = endtime - _time()
175
                    if remaining <= 0.0:
176
                        raise Empty
177
                    self.not_empty.wait(remaining)
178
            item = self._get()
179
            self.not_full.notify()
180
            return item
181
        finally:
182
            self.not_empty.release()
183
3e-06 sec 184
    def get_nowait(self):
185
        """Remove and return an item from the queue without blocking.
186
187
        Only get an item if one is immediately available. Otherwise
188
        raise the Empty exception.
189
        """
190
        return self.get(False)
191
192
    # Override these methods to implement other queue organizations
193
    # (e.g. stack or priority queue).
194
    # These will only be called with appropriate locks held
195
196
    # Initialize the queue representation
1.7e-05 sec 197
    def _init(self, maxsize):
2e-06 sec 198
        self.queue = deque()
199
3e-06 sec 200
    def _qsize(self, len=len):
201
        return len(self.queue)
202
203
    # Put a new item in the queue
7.8e-05 sec 204
    def _put(self, item):
2.1e-05 sec 205
        self.queue.append(item)
206
207
    # Get an item from the queue
2e-06 sec 208
    def _get(self):
209
        return self.queue.popleft()
210
211
1.2e-05 sec 212
class PriorityQueue(Queue):
213
    '''Variant of Queue that retrieves open entries in priority order (lowest first).
214
215
    Entries are typically tuples of the form:  (priority number, data).
2e-06 sec 216
    '''
217
1e-06 sec 218
    def _init(self, maxsize):
219
        self.queue = []
220
3e-06 sec 221
    def _qsize(self, len=len):
222
        return len(self.queue)
223
2e-06 sec 224
    def _put(self, item, heappush=heapq.heappush):
225
        heappush(self.queue, item)
226
3e-06 sec 227
    def _get(self, heappop=heapq.heappop):
228
        return heappop(self.queue)
229
230
1e-05 sec 231
class LifoQueue(Queue):
2e-06 sec 232
    '''Variant of Queue that retrieves most recently added entries first.'''
233
3e-06 sec 234
    def _init(self, maxsize):
235
        self.queue = []
236
2e-06 sec 237
    def _qsize(self, len=len):
238
        return len(self.queue)
239
3e-06 sec 240
    def _put(self, item):
241
        self.queue.append(item)
242
2e-06 sec 243
    def _get(self):
244
        return self.queue.pop()
/usr/lib/python2.6/SimpleHTTPServer.py
     time      num code
1
"""Simple HTTP Server.
2
3
This module builds on BaseHTTPServer by implementing the standard GET
4
and HEAD requests in a fairly straightforward manner.
5
0.000705 sec 6
"""
7
8
2e-06 sec 9
__version__ = "0.6"
10
4e-06 sec 11
__all__ = ["SimpleHTTPRequestHandler"]
12
2e-06 sec 13
import os
9e-06 sec 14
import posixpath
5e-06 sec 15
import BaseHTTPServer
2e-05 sec 16
import urllib
6e-06 sec 17
import cgi
6e-06 sec 18
import shutil
2e-05 sec 19
import mimetypes
1.6e-05 sec 20
try:
2e-06 sec 21
    from cStringIO import StringIO
22
except ImportError:
23
    from StringIO import StringIO
24
25
1.7e-05 sec 26
class SimpleHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
27
28
    """Simple HTTP request handler with GET and HEAD commands.
29
30
    This serves files from the current directory and any of its
31
    subdirectories.  The MIME type for files is determined by
32
    calling the .guess_type() method.
33
34
    The GET and HEAD requests are identical except that the HEAD
35
    request omits the actual contents of the file.
36
2e-06 sec 37
    """
38
2e-06 sec 39
    server_version = "SimpleHTTP/" + __version__
40
5e-06 sec 41
    def do_GET(self):
42
        """Serve a GET request."""
43
        f = self.send_head()
44
        if f:
45
            self.copyfile(f, self.wfile)
46
            f.close()
47
2e-06 sec 48
    def do_HEAD(self):
49
        """Serve a HEAD request."""
50
        f = self.send_head()
51
        if f:
52
            f.close()
53
3e-06 sec 54
    def send_head(self):
55
        """Common code for GET and HEAD commands.
56
57
        This sends the response code and MIME headers.
58
59
        Return value is either a file object (which has to be copied
60
        to the outputfile by the caller unless the command was HEAD,
61
        and must be closed by the caller under all circumstances), or
62
        None, in which case the caller has nothing further to do.
63
64
        """
65
        path = self.translate_path(self.path)
66
        f = None
67
        if os.path.isdir(path):
68
            if not self.path.endswith('/'):
69
                # redirect browser - doing basically what apache does
70
                self.send_response(301)
71
                self.send_header("Location", self.path + "/")
72
                self.end_headers()
73
                return None
74
            for index in "index.html", "index.htm":
75
                index = os.path.join(path, index)
76
                if os.path.exists(index):
77
                    path = index
78
                    break
79
            else:
80
                return self.list_directory(path)
81
        ctype = self.guess_type(path)
82
        try:
83
            # Always read in binary mode. Opening files in text mode may cause
84
            # newline translations, making the actual size of the content
85
            # transmitted *less* than the content-length!
86
            f = open(path, 'rb')
87
        except IOError:
88
            self.send_error(404, "File not found")
89
            return None
90
        self.send_response(200)
91
        self.send_header("Content-type", ctype)
92
        fs = os.fstat(f.fileno())
93
        self.send_header("Content-Length", str(fs[6]))
94
        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
95
        self.end_headers()
96
        return f
97
3e-06 sec 98
    def list_directory(self, path):
99
        """Helper to produce a directory listing (absent index.html).
100
101
        Return value is either a file object, or None (indicating an
102
        error).  In either case, the headers are sent, making the
103
        interface the same as for send_head().
104
105
        """
106
        try:
107
            list = os.listdir(path)
108
        except os.error:
109
            self.send_error(404, "No permission to list directory")
110
            return None
111
        list.sort(key=lambda a: a.lower())
112
        f = StringIO()
113
        displaypath = cgi.escape(urllib.unquote(self.path))
114
        f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
115
        f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath)
116
        f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
117
        f.write("<hr>\n<ul>\n")
118
        for name in list:
119
            fullname = os.path.join(path, name)
120
            displayname = linkname = name
121
            # Append / for directories or @ for symbolic links
122
            if os.path.isdir(fullname):
123
                displayname = name + "/"
124
                linkname = name + "/"
125
            if os.path.islink(fullname):
126
                displayname = name + "@"
127
                # Note: a link to a directory displays with @ and links with /
128
            f.write('<li><a href="%s">%s</a>\n'
129
                    % (urllib.quote(linkname), cgi.escape(displayname)))
130
        f.write("</ul>\n<hr>\n</body>\n</html>\n")
131
        length = f.tell()
132
        f.seek(0)
133
        self.send_response(200)
134
        self.send_header("Content-type", "text/html")
135
        self.send_header("Content-Length", str(length))
136
        self.end_headers()
137
        return f
138
2e-06 sec 139
    def translate_path(self, path):
140
        """Translate a /-separated PATH to the local filename syntax.
141
142
        Components that mean special things to the local file system
143
        (e.g. drive or directory names) are ignored.  (XXX They should
144
        probably be diagnosed.)
145
146
        """
147
        # abandon query parameters
148
        path = path.split('?',1)[0]
149
        path = path.split('#',1)[0]
150
        path = posixpath.normpath(urllib.unquote(path))
151
        words = path.split('/')
152
        words = filter(None, words)
153
        path = os.getcwd()
154
        for word in words:
155
            drive, word = os.path.splitdrive(word)
156
            head, word = os.path.split(word)
157
            if word in (os.curdir, os.pardir): continue
158
            path = os.path.join(path, word)
159
        return path
160
3e-06 sec 161
    def copyfile(self, source, outputfile):
162
        """Copy all data between two file objects.
163
164
        The SOURCE argument is a file object open for reading
165
        (or anything with a read() method) and the DESTINATION
166
        argument is a file object open for writing (or
167
        anything with a write() method).
168
169
        The only reason for overriding this would be to change
170
        the block size or perhaps to replace newlines by CRLF
171
        -- note however that this the default server uses this
172
        to copy binary data as well.
173
174
        """
175
        shutil.copyfileobj(source, outputfile)
176
2e-06 sec 177
    def guess_type(self, path):
178
        """Guess the type of a file.
179
180
        Argument is a PATH (a filename).
181
182
        Return value is a string of the form type/subtype,
183
        usable for a MIME Content-type header.
184
185
        The default implementation looks the file's extension
186
        up in the table self.extensions_map, using application/octet-stream
187
        as a default; however it would be permissible (if
188
        slow) to look inside the data to make a better guess.
189
190
        """
191
192
        base, ext = posixpath.splitext(path)
193
        if ext in self.extensions_map:
194
            return self.extensions_map[ext]
195
        ext = ext.lower()
196
        if ext in self.extensions_map:
197
            return self.extensions_map[ext]
198
        else:
199
            return self.extensions_map['']
200
2e-06 sec 201
    if not mimetypes.inited:
3e-06 sec 202
        mimetypes.init() # try to read system mime.types
4e-06 sec 203
    extensions_map = mimetypes.types_map.copy()
9.5e-05 sec 204
    extensions_map.update({
4e-06 sec 205
        '': 'application/octet-stream', # Default
2e-06 sec 206
        '.py': 'text/plain',
2e-06 sec 207
        '.c': 'text/plain',
3e-06 sec 208
        '.h': 'text/plain',
209
        })
210
211
2.1e-05 sec 212
def test(HandlerClass = SimpleHTTPRequestHandler,
2e-06 sec 213
         ServerClass = BaseHTTPServer.HTTPServer):
214
    BaseHTTPServer.test(HandlerClass, ServerClass)
215
216
6e-06 sec 217
if __name__ == '__main__':
218
    test()
/usr/lib/python2.6/SocketServer.py
     time      num code
1
"""Generic socket server classes.
2
3
This module tries to capture the various aspects of defining a server:
4
5
For socket-based servers:
6
7
- address family:
8
        - AF_INET{,6}: IP (Internet Protocol) sockets (default)
9
        - AF_UNIX: Unix domain sockets
10
        - others, e.g. AF_DECNET are conceivable (see <socket.h>
11
- socket type:
12
        - SOCK_STREAM (reliable stream, e.g. TCP)
13
        - SOCK_DGRAM (datagrams, e.g. UDP)
14
15
For request-based servers (including socket-based):
16
17
- client address verification before further looking at the request
18
        (This is actually a hook for any processing that needs to look
19
         at the request before anything else, e.g. logging)
20
- how to handle multiple requests:
21
        - synchronous (one request is handled at a time)
22
        - forking (each request is handled by a new process)
23
        - threading (each request is handled by a new thread)
24
25
The classes in this module favor the server type that is simplest to
26
write: a synchronous TCP/IP server.  This is bad class design, but
27
save some typing.  (There's also the issue that a deep class hierarchy
28
slows down method lookups.)
29
30
There are five classes in an inheritance diagram, four of which represent
31
synchronous servers of four types:
32
33
        +------------+
34
        | BaseServer |
35
        +------------+
36
              |
37
              v
38
        +-----------+        +------------------+
39
        | TCPServer |------->| UnixStreamServer |
40
        +-----------+        +------------------+
41
              |
42
              v
43
        +-----------+        +--------------------+
44
        | UDPServer |------->| UnixDatagramServer |
45
        +-----------+        +--------------------+
46
47
Note that UnixDatagramServer derives from UDPServer, not from
48
UnixStreamServer -- the only difference between an IP and a Unix
49
stream server is the address family, which is simply repeated in both
50
unix server classes.
51
52
Forking and threading versions of each type of server can be created
53
using the ForkingMixIn and ThreadingMixIn mix-in classes.  For
54
instance, a threading UDP server class is created as follows:
55
56
        class ThreadingUDPServer(ThreadingMixIn, UDPServer): pass
57
58
The Mix-in class must come first, since it overrides a method defined
59
in UDPServer! Setting the various member variables also changes
60
the behavior of the underlying server mechanism.
61
62
To implement a service, you must derive a class from
63
BaseRequestHandler and redefine its handle() method.  You can then run
64
various versions of the service by combining one of the server classes
65
with your request handler class.
66
67
The request handler class must be different for datagram or stream
68
services.  This can be hidden by using the request handler
69
subclasses StreamRequestHandler or DatagramRequestHandler.
70
71
Of course, you still have to use your head!
72
73
For instance, it makes no sense to use a forking server if the service
74
contains state in memory that can be modified by requests (since the
75
modifications in the child process would never reach the initial state
76
kept in the parent process and passed to each child).  In this case,
77
you can use a threading server, but you will probably have to use
78
locks to avoid two requests that come in nearly simultaneous to apply
79
conflicting changes to the server state.
80
81
On the other hand, if you are building e.g. an HTTP server, where all
82
data is stored externally (e.g. in the file system), a synchronous
83
class will essentially render the service "deaf" while one request is
84
being handled -- which may be for a very long time if a client is slow
85
to reqd all the data it has requested.  Here a threading or forking
86
server is appropriate.
87
88
In some cases, it may be appropriate to process part of a request
89
synchronously, but to finish processing in a forked child depending on
90
the request data.  This can be implemented by using a synchronous
91
server and doing an explicit fork in the request handler class
92
handle() method.
93
94
Another approach to handling multiple simultaneous requests in an
95
environment that supports neither threads nor fork (or where these are
96
too expensive or inappropriate for the service) is to maintain an
97
explicit table of partially finished requests and to use select() to
98
decide which request to work on next (or whether to handle a new
99
incoming request).  This is particularly important for stream services
100
where each client can potentially be connected for a long time (if
101
threads or subprocesses cannot be used).
102
103
Future work:
104
- Standard classes for Sun RPC (which uses either UDP or TCP)
105
- Standard mix-in classes to implement various authentication
106
  and encryption schemes
107
- Standard framework for select-based multiplexing
108
109
XXX Open problems:
110
- What to do with out-of-band data?
111
112
BaseServer:
113
- split generic "request" functionality out into BaseServer class.
114
  Copyright (C) 2000  Luke Kenneth Casson Leighton <lkcl@samba.org>
115
116
  example: read entries from a SQL database (requires overriding
117
  get_request() to return a table entry from the database).
118
  entry is processed by a RequestHandlerClass.
119
0.000876 sec 120
"""
121
122
# Author of the BaseServer patch: Luke Kenneth Casson Leighton
123
124
# XXX Warning!
125
# There is a test suite for this module, but it cannot be run by the
126
# standard regression test.
127
# To run it manually, run Lib/test/test_socketserver.py.
128
2e-06 sec 129
__version__ = "0.4"
130
131
3e-06 sec 132
import socket
8e-06 sec 133
import select
4e-06 sec 134
import sys
5e-06 sec 135
import os
4e-06 sec 136
try:
2e-06 sec 137
    import threading
138
except ImportError:
139
    import dummy_threading as threading
140
5e-06 sec 141
__all__ = ["TCPServer","UDPServer","ForkingUDPServer","ForkingTCPServer",
2e-06 sec 142
           "ThreadingUDPServer","ThreadingTCPServer","BaseRequestHandler",
2e-06 sec 143
           "StreamRequestHandler","DatagramRequestHandler",
2e-06 sec 144
           "ThreadingMixIn", "ForkingMixIn"]
3e-06 sec 145
if hasattr(socket, "AF_UNIX"):
7e-06 sec 146
    __all__.extend(["UnixStreamServer","UnixDatagramServer",
3e-06 sec 147
                    "ThreadingUnixStreamServer",
2e-06 sec 148
                    "ThreadingUnixDatagramServer"])
149
1.1e-05 sec 150
class BaseServer:
151
152
    """Base class for server classes.
153
154
    Methods for the caller:
155
156
    - __init__(server_address, RequestHandlerClass)
157
    - serve_forever(poll_interval=0.5)
158
    - shutdown()
159
    - handle_request()  # if you do not use serve_forever()
160
    - fileno() -> int   # for select()
161
162
    Methods that may be overridden:
163
164
    - server_bind()
165
    - server_activate()
166
    - get_request() -> request, client_address
167
    - handle_timeout()
168
    - verify_request(request, client_address)
169
    - server_close()
170
    - process_request(request, client_address)
171
    - close_request(request)
172
    - handle_error()
173
174
    Methods for derived classes:
175
176
    - finish_request(request, client_address)
177
178
    Class variables that may be overridden by derived classes or
179
    instances:
180
181
    - timeout
182
    - address_family
183
    - socket_type
184
    - allow_reuse_address
185
186
    Instance variables:
187
188
    - RequestHandlerClass
189
    - socket
190
2e-06 sec 191
    """
192
1e-06 sec 193
    timeout = None
194
2e-06 sec 195
    def __init__(self, server_address, RequestHandlerClass):
196
        """Constructor.  May be extended, do not override."""
197
        self.server_address = server_address
198
        self.RequestHandlerClass = RequestHandlerClass
199
        self.__is_shut_down = threading.Event()
200
        self.__serving = False
201
2e-06 sec 202
    def server_activate(self):
203
        """Called by constructor to activate the server.
204
205
        May be overridden.
206
207
        """
208
        pass
209
2e-06 sec 210
    def serve_forever(self, poll_interval=0.5):
211
        """Handle one request at a time until shutdown.
212
213
        Polls for shutdown every poll_interval seconds. Ignores
214
        self.timeout. If you need to do periodic tasks, do them in
215
        another thread.
216
        """
217
        self.__serving = True
218
        self.__is_shut_down.clear()
219
        while self.__serving:
220
            # XXX: Consider using another file descriptor or
221
            # connecting to the socket to wake this up instead of
222
            # polling. Polling reduces our responsiveness to a
223
            # shutdown request and wastes cpu at all other times.
224
            r, w, e = select.select([self], [], [], poll_interval)
225
            if r:
226
                self._handle_request_noblock()
227
        self.__is_shut_down.set()
228
4e-06 sec 229
    def shutdown(self):
230
        """Stops the serve_forever loop.
231
232
        Blocks until the loop has finished. This must be called while
233
        serve_forever() is running in another thread, or it will
234
        deadlock.
235
        """
236
        self.__serving = False
237
        self.__is_shut_down.wait()
238
239
    # The distinction between handling, getting, processing and
240
    # finishing a request is fairly arbitrary.  Remember:
241
    #
242
    # - handle_request() is the top-level call.  It calls
243
    #   select, get_request(), verify_request() and process_request()
244
    # - get_request() is different for stream or datagram sockets
245
    # - process_request() is the place that may fork a new process
246
    #   or create a new thread to finish the request
247
    # - finish_request() instantiates the request handler class;
248
    #   this constructor will handle the request all by itself
249
2e-06 sec 250
    def handle_request(self):
251
        """Handle one request, possibly blocking.
252
253
        Respects self.timeout.
254
        """
255
        # Support people who used socket.settimeout() to escape
256
        # handle_request before self.timeout was available.
257
        timeout = self.socket.gettimeout()
258
        if timeout is None:
259
            timeout = self.timeout
260
        elif self.timeout is not None:
261
            timeout = min(timeout, self.timeout)
262
        fd_sets = select.select([self], [], [], timeout)
263
        if not fd_sets[0]:
264
            self.handle_timeout()
265
            return
266
        self._handle_request_noblock()
267
3e-06 sec 268
    def _handle_request_noblock(self):
269
        """Handle one request, without blocking.
270
271
        I assume that select.select has returned that the socket is
272
        readable before this function was called, so there should be
273
        no risk of blocking in get_request().
274
        """
275
        try:
276
            request, client_address = self.get_request()
277
        except socket.error:
278
            return
279
        if self.verify_request(request, client_address):
280
            try:
281
                self.process_request(request, client_address)
282
            except:
283
                self.handle_error(request, client_address)
284
                self.close_request(request)
285
2e-06 sec 286
    def handle_timeout(self):
287
        """Called if no new request arrives within self.timeout.
288
289
        Overridden by ForkingMixIn.
290
        """
291
        pass
292
2e-06 sec 293
    def verify_request(self, request, client_address):
294
        """Verify the request.  May be overridden.
295
296
        Return True if we should proceed with this request.
297
298
        """
299
        return True
300
2e-06 sec 301
    def process_request(self, request, client_address):
302
        """Call finish_request.
303
304
        Overridden by ForkingMixIn and ThreadingMixIn.
305
306
        """
307
        self.finish_request(request, client_address)
308
        self.close_request(request)
309
2e-06 sec 310
    def server_close(self):
311
        """Called to clean-up the server.
312
313
        May be overridden.
314
315
        """
316
        pass
317
3e-06 sec 318
    def finish_request(self, request, client_address):
319
        """Finish one request by instantiating RequestHandlerClass."""
320
        self.RequestHandlerClass(request, client_address, self)
321
2e-06 sec 322
    def close_request(self, request):
323
        """Called to clean up an individual request."""
324
        pass
325
2e-06 sec 326
    def handle_error(self, request, client_address):
327
        """Handle an error gracefully.  May be overridden.
328
329
        The default is to print a traceback and continue.
330
331
        """
332
        print '-'*40
333
        print 'Exception happened during processing of request from',
334
        print client_address
335
        import traceback
336
        traceback.print_exc() # XXX But this goes to stderr!
337
        print '-'*40
338
339
1e-05 sec 340
class TCPServer(BaseServer):
341
342
    """Base class for various socket-based server classes.
343
344
    Defaults to synchronous IP stream (i.e., TCP).
345
346
    Methods for the caller:
347
348
    - __init__(server_address, RequestHandlerClass, bind_and_activate=True)
349
    - serve_forever(poll_interval=0.5)
350
    - shutdown()
351
    - handle_request()  # if you don't use serve_forever()
352
    - fileno() -> int   # for select()
353
354
    Methods that may be overridden:
355
356
    - server_bind()
357
    - server_activate()
358
    - get_request() -> request, client_address
359
    - handle_timeout()
360
    - verify_request(request, client_address)
361
    - process_request(request, client_address)
362
    - close_request(request)
363
    - handle_error()
364
365
    Methods for derived classes:
366
367
    - finish_request(request, client_address)
368
369
    Class variables that may be overridden by derived classes or
370
    instances:
371
372
    - timeout
373
    - address_family
374
    - socket_type
375
    - request_queue_size (only for stream sockets)
376
    - allow_reuse_address
377
378
    Instance variables:
379
380
    - server_address
381
    - RequestHandlerClass
382
    - socket
383
2e-06 sec 384
    """
385
2e-06 sec 386
    address_family = socket.AF_INET
387
3e-06 sec 388
    socket_type = socket.SOCK_STREAM
389
3e-06 sec 390
    request_queue_size = 5
391
2e-06 sec 392
    allow_reuse_address = False
393
3e-06 sec 394
    def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
395
        """Constructor.  May be extended, do not override."""
396
        BaseServer.__init__(self, server_address, RequestHandlerClass)
397
        self.socket = socket.socket(self.address_family,
398
                                    self.socket_type)
399
        if bind_and_activate:
400
            self.server_bind()
401
            self.server_activate()
402
2e-06 sec 403
    def server_bind(self):
404
        """Called by constructor to bind the socket.
405
406
        May be overridden.
407
408
        """
409
        if self.allow_reuse_address:
410
            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
411
        self.socket.bind(self.server_address)
412
        self.server_address = self.socket.getsockname()
413
2e-06 sec 414
    def server_activate(self):
415
        """Called by constructor to activate the server.
416
417
        May be overridden.
418
419
        """
420
        self.socket.listen(self.request_queue_size)
421
3e-06 sec 422
    def server_close(self):
423
        """Called to clean-up the server.
424
425
        May be overridden.
426
427
        """
428
        self.socket.close()
429
2e-06 sec 430
    def fileno(self):
431
        """Return socket file number.
432
433
        Interface required by select().
434
435
        """
436
        return self.socket.fileno()
437
2e-06 sec 438
    def get_request(self):
439
        """Get the request and client address from the socket.
440
441
        May be overridden.
442
443
        """
444
        return self.socket.accept()
445
3e-06 sec 446
    def close_request(self, request):
447
        """Called to clean up an individual request."""
448
        request.close()
449
450
1e-05 sec 451
class UDPServer(TCPServer):
452
2e-06 sec 453
    """UDP server class."""
454
1e-06 sec 455
    allow_reuse_address = False
456
3e-06 sec 457
    socket_type = socket.SOCK_DGRAM
458
3e-06 sec 459
    max_packet_size = 8192
460
2e-06 sec 461
    def get_request(self):
462
        data, client_addr = self.socket.recvfrom(self.max_packet_size)
463
        return (data, self.socket), client_addr
464
3e-06 sec 465
    def server_activate(self):
466
        # No need to call listen() for UDP.
467
        pass
468
2e-06 sec 469
    def close_request(self, request):
470
        # No need to close anything.
471
        pass
472
8e-06 sec 473
class ForkingMixIn:
474
1e-06 sec 475
    """Mix-in class to handle each request in a new process."""
476
2e-06 sec 477
    timeout = 300
2e-06 sec 478
    active_children = None
2e-06 sec 479
    max_children = 40
480
1e-06 sec 481
    def collect_children(self):
482
        """Internal routine to wait for children that have exited."""
483
        if self.active_children is None: return
484
        while len(self.active_children) >= self.max_children:
485
            # XXX: This will wait for any child process, not just ones
486
            # spawned by this library. This could confuse other
487
            # libraries that expect to be able to wait for their own
488
            # children.
489
            try:
490
                pid, status = os.waitpid(0, options=0)
491
            except os.error:
492
                pid = None
493
            if pid not in self.active_children: continue
494
            self.active_children.remove(pid)
495
496
        # XXX: This loop runs more system calls than it ought
497
        # to. There should be a way to put the active_children into a
498
        # process group and then use os.waitpid(-pgid) to wait for any
499
        # of that set, but I couldn't find a way to allocate pgids
500
        # that couldn't collide.
501
        for child in self.active_children:
502
            try:
503
                pid, status = os.waitpid(child, os.WNOHANG)
504
            except os.error:
505
                pid = None
506
            if not pid: continue
507
            try:
508
                self.active_children.remove(pid)
509
            except ValueError, e:
510
                raise ValueError('%s. x=%d and list=%r' % (e.message, pid,
511
                                                           self.active_children))
512
3e-06 sec 513
    def handle_timeout(self):
514
        """Wait for zombies after self.timeout seconds of inactivity.
515
516
        May be extended, do not override.
517
        """
518
        self.collect_children()
519
3e-06 sec 520
    def process_request(self, request, client_address):
521
        """Fork a new subprocess to process the request."""
522
        self.collect_children()
523
        pid = os.fork()
524
        if pid:
525
            # Parent process
526
            if self.active_children is None:
527
                self.active_children = []
528
            self.active_children.append(pid)
529
            self.close_request(request)
530
            return
531
        else:
532
            # Child process.
533
            # This must never return, hence os._exit()!
534
            try:
535
                self.finish_request(request, client_address)
536
                os._exit(0)
537
            except:
538
                try:
539
                    self.handle_error(request, client_address)
540
                finally:
541
                    os._exit(1)
542
543
8e-06 sec 544
class ThreadingMixIn:
3e-06 sec 545
    """Mix-in class to handle each request in a new thread."""
546
547
    # Decides how threads will act upon termination of the
548
    # main process
1e-06 sec 549
    daemon_threads = False
550
2e-06 sec 551
    def process_request_thread(self, request, client_address):
552
        """Same as in BaseServer but as a thread.
553
554
        In addition, exception handling is done here.
555
556
        """
557
        try:
558
            self.finish_request(request, client_address)
559
            self.close_request(request)
560
        except:
561
            self.handle_error(request, client_address)
562
            self.close_request(request)
563
3e-06 sec 564
    def process_request(self, request, client_address):
565
        """Start a new thread to process the request."""
566
        t = threading.Thread(target = self.process_request_thread,
567
                             args = (request, client_address))
568
        if self.daemon_threads:
569
            t.setDaemon (1)
570
        t.start()
571
572
9e-06 sec 573
class ForkingUDPServer(ForkingMixIn, UDPServer): pass
1e-05 sec 574
class ForkingTCPServer(ForkingMixIn, TCPServer): pass
575
1e-05 sec 576
class ThreadingUDPServer(ThreadingMixIn, UDPServer): pass
1.2e-05 sec 577
class ThreadingTCPServer(ThreadingMixIn, TCPServer): pass
578
7e-06 sec 579
if hasattr(socket, 'AF_UNIX'):
580
8e-06 sec 581
    class UnixStreamServer(TCPServer):
2e-06 sec 582
        address_family = socket.AF_UNIX
583
1.1e-05 sec 584
    class UnixDatagramServer(UDPServer):
2e-06 sec 585
        address_family = socket.AF_UNIX
586
1e-05 sec 587
    class ThreadingUnixStreamServer(ThreadingMixIn, UnixStreamServer): pass
588
1e-05 sec 589
    class ThreadingUnixDatagramServer(ThreadingMixIn, UnixDatagramServer): pass
590
9e-06 sec 591
class BaseRequestHandler:
592
593
    """Base class for request handler classes.
594
595
    This class is instantiated for each request to be handled.  The
596
    constructor sets the instance variables request, client_address
597
    and server, and then calls the handle() method.  To implement a
598
    specific service, all you need to do is to derive a class which
599
    defines a handle() method.
600
601
    The handle() method can find the request as self.request, the
602
    client address as self.client_address, and the server (in case it
603
    needs access to per-server information) as self.server.  Since a
604
    separate instance is created for each request, the handle() method
605
    can define arbitrary other instance variariables.
606
3e-06 sec 607
    """
608
1e-06 sec 609
    def __init__(self, request, client_address, server):
610
        self.request = request
611
        self.client_address = client_address
612
        self.server = server
613
        try:
614
            self.setup()
615
            self.handle()
616
            self.finish()
617
        finally:
618
            sys.exc_traceback = None    # Help garbage collection
619
2e-06 sec 620
    def setup(self):
621
        pass
622
3e-06 sec 623
    def handle(self):
624
        pass
625
2e-06 sec 626
    def finish(self):
627
        pass
628
629
630
# The following two classes make it possible to use the same service
631
# class for stream or datagram servers.
632
# Each class sets up these instance variables:
633
# - rfile: a file object from which receives the request is read
634
# - wfile: a file object to which the reply is written
635
# When the handle() method returns, wfile is flushed properly
636
637
8e-06 sec 638
class StreamRequestHandler(BaseRequestHandler):
639
2e-06 sec 640
    """Define self.rfile and self.wfile for stream sockets."""
641
642
    # Default buffer sizes for rfile, wfile.
643
    # We default rfile to buffered because otherwise it could be
644
    # really slow for large data (a getc() call per byte); we make
645
    # wfile unbuffered because (a) often after a write() we want to
646
    # read and we need to flush the line; (b) big writes to unbuffered
647
    # files are typically optimized by stdio even when big reads
648
    # aren't.
2e-06 sec 649
    rbufsize = -1
2e-06 sec 650
    wbufsize = 0
651
1e-06 sec 652
    def setup(self):
653
        self.connection = self.request
654
        self.rfile = self.connection.makefile('rb', self.rbufsize)
655
        self.wfile = self.connection.makefile('wb', self.wbufsize)
656
3e-06 sec 657
    def finish(self):
658
        if not self.wfile.closed:
659
            self.wfile.flush()
660
        self.wfile.close()
661
        self.rfile.close()
662
663
1.6e-05 sec 664
class DatagramRequestHandler(BaseRequestHandler):
665
666
    # XXX Regrettably, I cannot get this working on Linux;
667
    # s.recvfrom() doesn't return a meaningful client address.
668
2e-06 sec 669
    """Define self.rfile and self.wfile for datagram sockets."""
670
2e-06 sec 671
    def setup(self):
672
        try:
673
            from cStringIO import StringIO
674
        except ImportError:
675
            from StringIO import StringIO
676
        self.packet, self.socket = self.request
677
        self.rfile = StringIO(self.packet)
678
        self.wfile = StringIO()
679
2e-06 sec 680
    def finish(self):
681
        self.socket.sendto(self.wfile.getvalue(), self.client_address)
/usr/lib/python2.6/UserDict.py
     time      num code
1
"""A more or less complete user-defined wrapper around dictionary objects."""
2
3
class UserDict:
4
    def __init__(self, dict=None, **kwargs):
5
        self.data = {}
6
        if dict is not None:
7
            self.update(dict)
8
        if len(kwargs):
9
            self.update(kwargs)
10
    def __repr__(self): return repr(self.data)
11
    def __cmp__(self, dict):
12
        if isinstance(dict, UserDict):
13
            return cmp(self.data, dict.data)
14
        else:
15
            return cmp(self.data, dict)
16
    def __len__(self): return len(self.data)
17
    def __getitem__(self, key):
18
        if key in self.data:
19
            return self.data[key]
20
        if hasattr(self.__class__, "__missing__"):
21
            return self.__class__.__missing__(self, key)
22
        raise KeyError(key)
23
    def __setitem__(self, key, item): self.data[key] = item
24
    def __delitem__(self, key): del self.data[key]
25
    def clear(self): self.data.clear()
26
    def copy(self):
27
        if self.__class__ is UserDict:
28
            return UserDict(self.data.copy())
29
        import copy
30
        data = self.data
31
        try:
32
            self.data = {}
33
            c = copy.copy(self)
34
        finally:
35
            self.data = data
36
        c.update(self)
37
        return c
38
    def keys(self): return self.data.keys()
39
    def items(self): return self.data.items()
40
    def iteritems(self): return self.data.iteritems()
41
    def iterkeys(self): return self.data.iterkeys()
42
    def itervalues(self): return self.data.itervalues()
43
    def values(self): return self.data.values()
3.2e-05 sec 44
    def has_key(self, key): return key in self.data
45
    def update(self, dict=None, **kwargs):
46
        if dict is None:
47
            pass
48
        elif isinstance(dict, UserDict):
49
            self.data.update(dict.data)
50
        elif isinstance(dict, type({})) or not hasattr(dict, 'items'):
51
            self.data.update(dict)
52
        else:
53
            for k, v in dict.items():
54
                self[k] = v
55
        if len(kwargs):
56
            self.data.update(kwargs)
57
    def get(self, key, failobj=None):
58
        if key not in self:
59
            return failobj
60
        return self[key]
61
    def setdefault(self, key, failobj=None):
62
        if key not in self:
63
            self[key] = failobj
64
        return self[key]
65
    def pop(self, key, *args):
66
        return self.data.pop(key, *args)
67
    def popitem(self):
68
        return self.data.popitem()
69
    def __contains__(self, key):
70
        return key in self.data
71
    @classmethod
72
    def fromkeys(cls, iterable, value=None):
73
        d = cls()
74
        for key in iterable:
75
            d[key] = value
76
        return d
77
78
class IterableUserDict(UserDict):
79
    def __iter__(self):
80
        return iter(self.data)
81
82
import _abcoll
83
_abcoll.MutableMapping.register(IterableUserDict)
84
85
86
class DictMixin:
87
    # Mixin defining all dictionary methods for classes that already have
88
    # a minimum dictionary interface including getitem, setitem, delitem,
89
    # and keys. Without knowledge of the subclass constructor, the mixin
90
    # does not define __init__() or copy().  In addition to the four base
91
    # methods, progressively more efficiency comes with defining
92
    # __contains__(), __iter__(), and iteritems().
93
94
    # second level definitions support higher levels
95
    def __iter__(self):
96
        for k in self.keys():
97
            yield k
98
    def has_key(self, key):
99
        try:
100
            value = self[key]
101
        except KeyError:
102
            return False
103
        return True
104
    def __contains__(self, key):
105
        return self.has_key(key)
106
107
    # third level takes advantage of second level definitions
108
    def iteritems(self):
109
        for k in self:
110
            yield (k, self[k])
111
    def iterkeys(self):
112
        return self.__iter__()
113
114
    # fourth level uses definitions from lower levels
115
    def itervalues(self):
116
        for _, v in self.iteritems():
117
            yield v
118
    def values(self):
119
        return [v for _, v in self.iteritems()]
120
    def items(self):
121
        return list(self.iteritems())
122
    def clear(self):
123
        for key in self.keys():
124
            del self[key]
125
    def setdefault(self, key, default=None):
126
        try:
127
            return self[key]
128
        except KeyError:
129
            self[key] = default
130
        return default
131
    def pop(self, key, *args):
132
        if len(args) > 1:
133
            raise TypeError, "pop expected at most 2 arguments, got "\
134
                              + repr(1 + len(args))
135
        try:
136
            value = self[key]
137
        except KeyError:
138
            if args:
139
                return args[0]
140
            raise
141
        del self[key]
142
        return value
143
    def popitem(self):
144
        try:
145
            k, v = self.iteritems().next()
146
        except StopIteration:
147
            raise KeyError, 'container is empty'
148
        del self[k]
149
        return (k, v)
150
    def update(self, other=None, **kwargs):
151
        # Make progressively weaker assumptions about "other"
152
        if other is None:
153
            pass
154
        elif hasattr(other, 'iteritems'):  # iteritems saves memory and lookups
155
            for k, v in other.iteritems():
156
                self[k] = v
157
        elif hasattr(other, 'keys'):
158
            for k in other.keys():
159
                self[k] = other[k]
160
        else:
161
            for k, v in other:
162
                self[k] = v
163
        if kwargs:
164
            self.update(kwargs)
165
    def get(self, key, default=None):
166
        try:
167
            return self[key]
168
        except KeyError:
169
            return default
170
    def __repr__(self):
171
        return repr(dict(self.iteritems()))
172
    def __cmp__(self, other):
173
        if other is None:
174
            return 1
175
        if isinstance(other, DictMixin):
176
            other = dict(other.iteritems())
177
        return cmp(dict(self.iteritems()), other)
178
    def __len__(self):
179
        return len(self.keys())
/usr/lib/python2.6/genericpath.py
     time      num code
1
"""
2
Path operations common to more than one OS
3
Do not use directly.  The OS specific modules import the appropriate
4
functions from this module themselves.
5
"""
6
import os
7
import stat
8
9
__all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
10
           'getsize', 'isdir', 'isfile']
11
12
13
# Does a path exist?
14
# This is false for dangling symbolic links on systems that support them.
15
def exists(path):
16
    """Test whether a path exists.  Returns False for broken symbolic links"""
17
    try:
18
        st = os.stat(path)
19
    except os.error:
20
        return False
21
    return True
22
23
24
# This follows symbolic links, so both islink() and isdir() can be true
25
# for the same path ono systems that support symlinks
5.4e-05 sec 26
def isfile(path):
27
    """Test whether a path is a regular file"""
1.7e-05 sec 28
    try:
1.7e-05 sec 29
        st = os.stat(path)
0.000164 sec 30
    except os.error:
3.3e-05 sec 31
        return False
2.8e-05 sec 32
    return stat.S_ISREG(st.st_mode)
33
34
35
# Is a path a directory?
36
# This follows symbolic links, so both islink() and isdir()
37
# can be true for the same path on systems that support symlinks
38
def isdir(s):
39
    """Return true if the pathname refers to an existing directory."""
40
    try:
41
        st = os.stat(s)
42
    except os.error:
43
        return False
44
    return stat.S_ISDIR(st.st_mode)
45
46
47
def getsize(filename):
48
    """Return the size of a file, reported by os.stat()."""
49
    return os.stat(filename).st_size
50
51
52
def getmtime(filename):
53
    """Return the last modification time of a file, reported by os.stat()."""
54
    return os.stat(filename).st_mtime
55
56
57
def getatime(filename):
58
    """Return the last access time of a file, reported by os.stat()."""
59
    return os.stat(filename).st_atime
60
61
62
def getctime(filename):
63
    """Return the metadata change time of a file, reported by os.stat()."""
64
    return os.stat(filename).st_ctime
65
66
67
# Return the longest prefix of all list elements.
68
def commonprefix(m):
69
    "Given a list of pathnames, returns the longest common leading component"
70
    if not m: return ''
71
    s1 = min(m)
72
    s2 = max(m)
73
    for i, c in enumerate(s1):
74
        if c != s2[i]:
75
            return s1[:i]
76
    return s1
77
78
# Split a path in root and extension.
79
# The extension is everything starting at the last dot in the last
80
# pathname component; the root is everything before that.
81
# It is always true that root + ext == p.
82
83
# Generic implementation of splitext, to be parametrized with
84
# the separators
85
def _splitext(p, sep, altsep, extsep):
86
    """Split the extension from a pathname.
87
88
    Extension is everything from the last dot to the end, ignoring
89
    leading dots.  Returns "(root, ext)"; ext may be empty."""
90
91
    sepIndex = p.rfind(sep)
92
    if altsep:
93
        altsepIndex = p.rfind(altsep)
94
        sepIndex = max(sepIndex, altsepIndex)
95
96
    dotIndex = p.rfind(extsep)
97
    if dotIndex > sepIndex:
98
        # skip all leading dots
99
        filenameIndex = sepIndex + 1
100
        while filenameIndex < dotIndex:
101
            if p[filenameIndex] != extsep:
102
                return p[:dotIndex], p[dotIndex:]
103
            filenameIndex += 1
104
105
    return p, ''
/usr/lib/python2.6/heapq.py
     time      num code
1
# -*- coding: Latin-1 -*-
2
3
"""Heap queue algorithm (a.k.a. priority queue).
4
5
Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
6
all k, counting elements from 0.  For the sake of comparison,
7
non-existing elements are considered to be infinite.  The interesting
8
property of a heap is that a[0] is always its smallest element.
9
10
Usage:
11
12
heap = []            # creates an empty heap
13
heappush(heap, item) # pushes a new item on the heap
14
item = heappop(heap) # pops the smallest item from the heap
15
item = heap[0]       # smallest item on the heap without popping it
16
heapify(x)           # transforms list into a heap, in-place, in linear time
17
item = heapreplace(heap, item) # pops and returns smallest item, and adds
18
                               # new item; the heap size is unchanged
19
20
Our API differs from textbook heap algorithms as follows:
21
22
- We use 0-based indexing.  This makes the relationship between the
23
  index for a node and the indexes for its children slightly less
24
  obvious, but is more suitable since Python uses 0-based indexing.
25
26
- Our heappop() method returns the smallest item, not the largest.
27
28
These two make it possible to view the heap as a regular Python list
29
without surprises: heap[0] is the smallest item, and heap.sort()
30
maintains the heap invariant!
0.000613 sec 31
"""
32
33
# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
34
35
__about__ = """Heap queues
36
37
Unicode Decoding Error!!!
38
39
Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
40
all k, counting elements from 0.  For the sake of comparison,
41
non-existing elements are considered to be infinite.  The interesting
42
property of a heap is that a[0] is always its smallest element.
43
44
The strange invariant above is meant to be an efficient memory
45
representation for a tournament.  The numbers below are `k', not a[k]:
46
47
                                   0
48
49
                  1                                 2
50
51
          3               4                5               6
52
53
      7       8       9       10      11      12      13      14
54
55
    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
56
57
58
In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
59
an usual binary tournament we see in sports, each cell is the winner
60
over the two cells it tops, and we can trace the winner down the tree
61
to see all opponents s/he had.  However, in many computer applications
62
of such tournaments, we do not need to trace the history of a winner.
63
To be more memory efficient, when a winner is promoted, we try to
64
replace it by something else at a lower level, and the rule becomes
65
that a cell and the two cells it tops contain three different items,
66
but the top cell "wins" over the two topped cells.
67
68
If this heap invariant is protected at all time, index 0 is clearly
69
the overall winner.  The simplest algorithmic way to remove it and
70
find the "next" winner is to move some loser (let's say cell 30 in the
71
diagram above) into the 0 position, and then percolate this new 0 down
72
the tree, exchanging values, until the invariant is re-established.
73
This is clearly logarithmic on the total number of items in the tree.
74
By iterating over all items, you get an O(n ln n) sort.
75
76
A nice feature of this sort is that you can efficiently insert new
77
items while the sort is going on, provided that the inserted items are
78
not "better" than the last 0'th element you extracted.  This is
79
especially useful in simulation contexts, where the tree holds all
80
incoming events, and the "win" condition means the smallest scheduled
81
time.  When an event schedule other events for execution, they are
82
scheduled into the future, so they can easily go into the heap.  So, a
83
heap is a good structure for implementing schedulers (this is what I
84
used for my MIDI sequencer :-).
85
86
Various structures for implementing schedulers have been extensively
87
studied, and heaps are good for this, as they are reasonably speedy,
88
the speed is almost constant, and the worst case is not much different
89
than the average case.  However, there are other representations which
90
are more efficient overall, yet the worst cases might be terrible.
91
92
Heaps are also very useful in big disk sorts.  You most probably all
93
know that a big sort implies producing "runs" (which are pre-sorted
94
sequences, which size is usually related to the amount of CPU memory),
95
followed by a merging passes for these runs, which merging is often
96
very cleverly organised[1].  It is very important that the initial
97
sort produces the longest runs possible.  Tournaments are a good way
98
to that.  If, using all the memory available to hold a tournament, you
99
replace and percolate items that happen to fit the current run, you'll
100
produce runs which are twice the size of the memory for random input,
101
and much better for input fuzzily ordered.
102
103
Moreover, if you output the 0'th item on disk and get an input which
104
may not fit in the current tournament (because the value "wins" over
105
the last output value), it cannot fit in the heap, so the size of the
106
heap decreases.  The freed memory could be cleverly reused immediately
107
for progressively building a second heap, which grows at exactly the
108
same rate the first heap is melting.  When the first heap completely
109
vanishes, you switch heaps and start a new run.  Clever and quite
110
effective!
111
112
In a word, heaps are useful memory structures to know.  I use them in
113
a few applications, and I think it is good to keep a `heap' module
114
around. :-)
115
116
--------------------
117
[1] The disk balancing algorithms which are current, nowadays, are
118
more annoying than clever, and this is a consequence of the seeking
119
capabilities of the disks.  On devices which cannot seek, like big
120
tape drives, the story was quite different, and one had to be very
121
clever to ensure (far in advance) that each tape movement will be the
122
most effective possible (that is, will best participate at
123
"progressing" the merge).  Some tapes were even able to read
124
backwards, and this was also used to avoid the rewinding time.
125
Believe me, real good tape sorts were quite spectacular to watch!
126
From all times, sorting has always been a Great Art! :-)
2e-06 sec 127
"""
128
4e-06 sec 129
__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
2e-06 sec 130
           'nlargest', 'nsmallest', 'heappushpop']
131
3e-06 sec 132
from itertools import islice, repeat, count, imap, izip, tee
2e-05 sec 133
from operator import itemgetter, neg
1e-05 sec 134
import bisect
135
6e-06 sec 136
def heappush(heap, item):
137
    """Push item onto heap, maintaining the heap invariant."""
138
    heap.append(item)
139
    _siftdown(heap, 0, len(heap)-1)
140
3e-06 sec 141
def heappop(heap):
142
    """Pop the smallest item off the heap, maintaining the heap invariant."""
143
    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
144
    if heap:
145
        returnitem = heap[0]
146
        heap[0] = lastelt
147
        _siftup(heap, 0)
148
    else:
149
        returnitem = lastelt
150
    return returnitem
151
2e-06 sec 152
def heapreplace(heap, item):
153
    """Pop and return the current smallest value, and add the new item.
154
155
    This is more efficient than heappop() followed by heappush(), and can be
156
    more appropriate when using a fixed-size heap.  Note that the value
157
    returned may be larger than item!  That constrains reasonable uses of
158
    this routine unless written as part of a conditional replacement:
159
160
        if item > heap[0]:
161
            item = heapreplace(heap, item)
162
    """
163
    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
164
    heap[0] = item
165
    _siftup(heap, 0)
166
    return returnitem
167
3e-06 sec 168
def heappushpop(heap, item):
169
    """Fast version of a heappush followed by a heappop."""
170
    if heap and heap[0] < item:
171
        item, heap[0] = heap[0], item
172
        _siftup(heap, 0)
173
    return item
174
2e-06 sec 175
def heapify(x):
176
    """Transform list into a heap, in-place, in O(len(heap)) time."""
177
    n = len(x)
178
    # Transform bottom-up.  The largest index there's any point to looking at
179
    # is the largest with a child index in-range, so must have 2*i + 1 < n,
180
    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
181
    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
182
    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
183
    for i in reversed(xrange(n//2)):
184
        _siftup(x, i)
185
2e-06 sec 186
def nlargest(n, iterable):
187
    """Find the n largest elements in a dataset.
188
189
    Equivalent to:  sorted(iterable, reverse=True)[:n]
190
    """
191
    it = iter(iterable)
192
    result = list(islice(it, n))
193
    if not result:
194
        return result
195
    heapify(result)
196
    _heappushpop = heappushpop
197
    for elem in it:
198
        heappushpop(result, elem)
199
    result.sort(reverse=True)
200
    return result
201
6e-06 sec 202
def nsmallest(n, iterable):
203
    """Find the n smallest elements in a dataset.
204
205
    Equivalent to:  sorted(iterable)[:n]
206
    """
207
    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
208
        # For smaller values of n, the bisect method is faster than a minheap.
209
        # It is also memory efficient, consuming only n elements of space.
210
        it = iter(iterable)
211
        result = sorted(islice(it, 0, n))
212
        if not result:
213
            return result
214
        insort = bisect.insort
215
        pop = result.pop
216
        los = result[-1]    # los --> Largest of the nsmallest
217
        for elem in it:
218
            if los <= elem:
219
                continue
220
            insort(result, elem)
221
            pop()
222
            los = result[-1]
223
        return result
224
    # An alternative approach manifests the whole iterable in memory but
225
    # saves comparisons by heapifying all at once.  Also, saves time
226
    # over bisect.insort() which has O(n) data movement time for every
227
    # insertion.  Finding the n smallest of an m length iterable requires
228
    #    O(m) + O(n log m) comparisons.
229
    h = list(iterable)
230
    heapify(h)
231
    return map(heappop, repeat(h, min(n, len(h))))
232
233
# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
234
# is the index of a leaf with a possibly out-of-order value.  Restore the
235
# heap invariant.
3e-06 sec 236
def _siftdown(heap, startpos, pos):
237
    newitem = heap[pos]
238
    # Follow the path to the root, moving parents down until finding a place
239
    # newitem fits.
240
    while pos > startpos:
241
        parentpos = (pos - 1) >> 1
242
        parent = heap[parentpos]
243
        if newitem < parent:
244
            heap[pos] = parent
245
            pos = parentpos
246
            continue
247
        break
248
    heap[pos] = newitem
249
250
# The child indices of heap index pos are already heaps, and we want to make
251
# a heap at index pos too.  We do this by bubbling the smaller child of
252
# pos up (and so on with that child's children, etc) until hitting a leaf,
253
# then using _siftdown to move the oddball originally at index pos into place.
254
#
255
# We *could* break out of the loop as soon as we find a pos where newitem <=
256
# both its children, but turns out that's not a good idea, and despite that
257
# many books write the algorithm that way.  During a heap pop, the last array
258
# element is sifted in, and that tends to be large, so that comparing it
259
# against values starting from the root usually doesn't pay (= usually doesn't
260
# get us out of the loop early).  See Knuth, Volume 3, where this is
261
# explained and quantified in an exercise.
262
#
263
# Cutting the # of comparisons is important, since these routines have no
264
# way to extract "the priority" from an array element, so that intelligence
265
# is likely to be hiding in custom __cmp__ methods, or in array elements
266
# storing (priority, record) tuples.  Comparisons are thus potentially
267
# expensive.
268
#
269
# On random arrays of length 1000, making this change cut the number of
270
# comparisons made by heapify() a little, and those made by exhaustive
271
# heappop() a lot, in accord with theory.  Here are typical results from 3
272
# runs (3 just to demonstrate how small the variance is):
273
#
274
# Compares needed by heapify     Compares needed by 1000 heappops
275
# --------------------------     --------------------------------
276
# 1837 cut to 1663               14996 cut to 8680
277
# 1855 cut to 1659               14966 cut to 8678
278
# 1847 cut to 1660               15024 cut to 8703
279
#
280
# Building the heap by using heappush() 1000 times instead required
281
# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
282
# you can use it.
283
#
284
# The total compares needed by list.sort() on the same lists were 8627,
285
# 8627, and 8632 (this should be compared to the sum of heapify() and
286
# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
287
# for sorting.
288
2e-06 sec 289
def _siftup(heap, pos):
290
    endpos = len(heap)
291
    startpos = pos
292
    newitem = heap[pos]
293
    # Bubble up the smaller child until hitting a leaf.
294
    childpos = 2*pos + 1    # leftmost child position
295
    while childpos < endpos:
296
        # Set childpos to index of smaller child.
297
        rightpos = childpos + 1
298
        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
299
            childpos = rightpos
300
        # Move the smaller child up.
301
        heap[pos] = heap[childpos]
302
        pos = childpos
303
        childpos = 2*pos + 1
304
    # The leaf at pos is empty now.  Put newitem there, and bubble it up
305
    # to its final resting place (by sifting its parents down).
306
    heap[pos] = newitem
307
    _siftdown(heap, startpos, pos)
308
309
# If available, use C implementation
2e-06 sec 310
try:
3e-06 sec 311
    from _heapq import heappush, heappop, heapify, heapreplace, nlargest, nsmallest, heappushpop
312
except ImportError:
313
    pass
314
0.000741 sec 315
def merge(*iterables):
316
    '''Merge multiple sorted inputs into a single sorted output.
317
318
    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
319
    does not pull the data into memory all at once, and assumes that each of
320
    the input streams is already sorted (smallest to largest).
321
322
    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
323
    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
324
325
    '''
326
    _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
327
328
    h = []
329
    h_append = h.append
330
    for itnum, it in enumerate(map(iter, iterables)):
331
        try:
332
            next = it.next
333
            h_append([next(), itnum, next])
334
        except _StopIteration:
335
            pass
336
    heapify(h)
337
338
    while 1:
339
        try:
340
            while 1:
341
                v, itnum, next = s = h[0]   # raises IndexError when h is empty
342
                yield v
343
                s[0] = next()               # raises StopIteration when exhausted
344
                _heapreplace(h, s)          # restore heap condition
345
        except _StopIteration:
346
            _heappop(h)                     # remove empty iterator
347
        except IndexError:
348
            return
349
350
# Extend the implementations of nsmallest and nlargest to use a key= argument
3e-06 sec 351
_nsmallest = nsmallest
3e-06 sec 352
def nsmallest(n, iterable, key=None):
353
    """Find the n smallest elements in a dataset.
354
355
    Equivalent to:  sorted(iterable, key=key)[:n]
356
    """
357
    if key is None:
358
        it = izip(iterable, count())                        # decorate
359
        result = _nsmallest(n, it)
360
        return map(itemgetter(0), result)                   # undecorate
361
    in1, in2 = tee(iterable)
362
    it = izip(imap(key, in1), count(), in2)                 # decorate
363
    result = _nsmallest(n, it)
364
    return map(itemgetter(2), result)                       # undecorate
365
4e-06 sec 366
_nlargest = nlargest
2e-06 sec 367
def nlargest(n, iterable, key=None):
368
    """Find the n largest elements in a dataset.
369
370
    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
371
    """
372
    if key is None:
373
        it = izip(iterable, imap(neg, count()))             # decorate
374
        result = _nlargest(n, it)
375
        return map(itemgetter(0), result)                   # undecorate
376
    in1, in2 = tee(iterable)
377
    it = izip(imap(key, in1), imap(neg, count()), in2)      # decorate
378
    result = _nlargest(n, it)
379
    return map(itemgetter(2), result)                       # undecorate
380
2e-06 sec 381
if __name__ == "__main__":
382
    # Simple sanity test
383
    heap = []
384
    data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
385
    for item in data:
386
        heappush(heap, item)
387
    sort = []
388
    while heap:
389
        sort.append(heappop(heap))
390
    print sort
391
392
    import doctest
393
    doctest.testmod()
/usr/lib/python2.6/mimetypes.py
     time      num code
1
"""Guess the MIME type of a file.
2
3
This module defines two useful functions:
4
5
guess_type(url, strict=1) -- guess the MIME type and encoding of a URL.
6
7
guess_extension(type, strict=1) -- guess the extension for a given MIME type.
8
9
It also contains the following, for tuning the behavior:
10
11
Data:
12
13
knownfiles -- list of files to parse
14
inited -- flag set when init() has been called
15
suffix_map -- dictionary mapping suffixes to suffixes
16
encodings_map -- dictionary mapping suffixes to encodings
17
types_map -- dictionary mapping suffixes to types
18
19
Functions:
20
21
init([files]) -- parse a list of files, default knownfiles
22
read_mime_types(file) -- parse one file, return a dictionary or None
0.000798 sec 23
"""
24
3e-06 sec 25
import os
1e-05 sec 26
import posixpath
5e-06 sec 27
import urllib
28
29
__all__ = [
5e-06 sec 30
    "guess_type","guess_extension","guess_all_extensions",
2e-06 sec 31
    "add_type","read_mime_types","init"
32
]
33
34
knownfiles = [
3e-06 sec 35
    "/etc/mime.types",
1e-06 sec 36
    "/etc/httpd/mime.types",                    # Mac OS X
2e-06 sec 37
    "/etc/httpd/conf/mime.types",               # Apache
2e-06 sec 38
    "/etc/apache/mime.types",                   # Apache 1
2e-06 sec 39
    "/etc/apache2/mime.types",                  # Apache 2
1e-06 sec 40
    "/usr/local/etc/httpd/conf/mime.types",
2e-06 sec 41
    "/usr/local/lib/netscape/mime.types",
2e-06 sec 42
    "/usr/local/etc/httpd/conf/mime.types",     # Apache 1.2
1e-06 sec 43
    "/usr/local/etc/mime.types",                # Apache 1.3
44
    ]
45
5e-06 sec 46
inited = False
47
48
1.1e-05 sec 49
class MimeTypes:
50
    """MIME-types datastore.
51
52
    This datastore can handle information from mime.types-style files
53
    and supports basic determination of MIME type from a filename or
54
    URL, and can guess a reasonable extension given a MIME type.
3e-06 sec 55
    """
56
1.2e-05 sec 57
    def __init__(self, filenames=(), strict=True):
2e-06 sec 58
        if not inited:
59
            init()
2e-06 sec 60
        self.encodings_map = encodings_map.copy()
7e-06 sec 61
        self.suffix_map = suffix_map.copy()
3e-06 sec 62
        self.types_map = ({}, {}) # dict for (non-strict, strict)
2e-06 sec 63
        self.types_map_inv = ({}, {})
0.000246 sec 64
        for (ext, type) in types_map.items():
0.00026 sec 65
            self.add_type(type, ext, True)
3.1e-05 sec 66
        for (ext, type) in common_types.items():
1.8e-05 sec 67
            self.add_type(type, ext, False)
3e-06 sec 68
        for name in filenames:
69
            self.read(name, strict)
70
0.003717 sec 71
    def add_type(self, type, ext, strict=True):
72
        """Add a mapping between a type and an extension.
73
74
        When the extension is already known, the new
75
        type will replace the old one. When the type
76
        is already known the extension will be added
77
        to the list of known extensions.
78
79
        If strict is true, information will be added to
80
        list of standard types, else to the list of non-standard
81
        types.
82
        """
0.001319 sec 83
        self.types_map[strict][ext] = type
0.00169 sec 84
        exts = self.types_map_inv[strict].setdefault(type, [])
0.00228 sec 85
        if ext not in exts:
0.001049 sec 86
            exts.append(ext)
87
3e-06 sec 88
    def guess_type(self, url, strict=True):
89
        """Guess the type of a file based on its URL.
90
91
        Return value is a tuple (type, encoding) where type is None if
92
        the type can't be guessed (no or unknown suffix) or a string
93
        of the form type/subtype, usable for a MIME Content-type
94
        header; and encoding is None for no encoding or the name of
95
        the program used to encode (e.g. compress or gzip).  The
96
        mappings are table driven.  Encoding suffixes are case
97
        sensitive; type suffixes are first tried case sensitive, then
98
        case insensitive.
99
100
        The suffixes .tgz, .taz and .tz (case sensitive!) are all
101
        mapped to '.tar.gz'.  (This is table-driven too, using the
102
        dictionary suffix_map.)
103
104
        Optional `strict' argument when False adds a bunch of commonly found,
105
        but non-standard types.
106
        """
107
        scheme, url = urllib.splittype(url)
108
        if scheme == 'data':
109
            # syntax of data URLs:
110
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
111
            # mediatype := [ type "/" subtype ] *( ";" parameter )
112
            # data      := *urlchar
113
            # parameter := attribute "=" value
114
            # type/subtype defaults to "text/plain"
115
            comma = url.find(',')
116
            if comma < 0:
117
                # bad data URL
118
                return None, None
119
            semi = url.find(';', 0, comma)
120
            if semi >= 0:
121
                type = url[:semi]
122
            else:
123
                type = url[:comma]
124
            if '=' in type or '/' not in type:
125
                type = 'text/plain'
126
            return type, None           # never compressed, so encoding is None
127
        base, ext = posixpath.splitext(url)
128
        while ext in self.suffix_map:
129
            base, ext = posixpath.splitext(base + self.suffix_map[ext])
130
        if ext in self.encodings_map:
131
            encoding = self.encodings_map[ext]
132
            base, ext = posixpath.splitext(base)
133
        else:
134
            encoding = None
135
        types_map = self.types_map[True]
136
        if ext in types_map:
137
            return types_map[ext], encoding
138
        elif ext.lower() in types_map:
139
            return types_map[ext.lower()], encoding
140
        elif strict:
141
            return None, encoding
142
        types_map = self.types_map[False]
143
        if ext in types_map:
144
            return types_map[ext], encoding
145
        elif ext.lower() in types_map:
146
            return types_map[ext.lower()], encoding
147
        else:
148
            return None, encoding
149
3e-06 sec 150
    def guess_all_extensions(self, type, strict=True):
151
        """Guess the extensions for a file based on its MIME type.
152
153
        Return value is a list of strings giving the possible filename
154
        extensions, including the leading dot ('.').  The extension is not
155
        guaranteed to have been associated with any particular data stream,
156
        but would be mapped to the MIME type `type' by guess_type().
157
158
        Optional `strict' argument when false adds a bunch of commonly found,
159
        but non-standard types.
160
        """
161
        type = type.lower()
162
        extensions = self.types_map_inv[True].get(type, [])
163
        if not strict:
164
            for ext in self.types_map_inv[False].get(type, []):
165
                if ext not in extensions:
166
                    extensions.append(ext)
167
        return extensions
168
3e-06 sec 169
    def guess_extension(self, type, strict=True):
170
        """Guess the extension for a file based on its MIME type.
171
172
        Return value is a string giving a filename extension,
173
        including the leading dot ('.').  The extension is not
174
        guaranteed to have been associated with any particular data
175
        stream, but would be mapped to the MIME type `type' by
176
        guess_type().  If no extension can be guessed for `type', None
177
        is returned.
178
179
        Optional `strict' argument when false adds a bunch of commonly found,
180
        but non-standard types.
181
        """
182
        extensions = self.guess_all_extensions(type, strict)
183
        if not extensions:
184
            return None
185
        return extensions[0]
186
3e-06 sec 187
    def read(self, filename, strict=True):
188
        """
189
        Read a single mime.types-format file, specified by pathname.
190
191
        If strict is true, information will be added to
192
        list of standard types, else to the list of non-standard
193
        types.
194
        """
195
        fp = open(filename)
196
        self.readfp(fp, strict)
197
        fp.close()
198
2.9e-05 sec 199
    def readfp(self, fp, strict=True):
200
        """
201
        Read a single mime.types-format file.
202
203
        If strict is true, information will be added to
204
        list of standard types, else to the list of non-standard
205
        types.
206
        """
2e-06 sec 207
        while 1:
0.001749 sec 208
            line = fp.readline()
0.002889 sec 209
            if not line:
2e-06 sec 210
                break
0.00152 sec 211
            words = line.split()
0.006116 sec 212
            for i in range(len(words)):
0.003603 sec 213
                if words[i][0] == '#':
6.7e-05 sec 214
                    del words[i:]
7e-05 sec 215
                    break
0.001935 sec 216
            if not words:
7.7e-05 sec 217
                continue
0.001533 sec 218
            type, suffixes = words[0], words[1:]
0.003087 sec 219
            for suff in suffixes:
0.001021 sec 220
                self.add_type(type, '.' + suff, strict)
221
8e-06 sec 222
def guess_type(url, strict=True):
223
    """Guess the type of a file based on its URL.
224
225
    Return value is a tuple (type, encoding) where type is None if the
226
    type can't be guessed (no or unknown suffix) or a string of the
227
    form type/subtype, usable for a MIME Content-type header; and
228
    encoding is None for no encoding or the name of the program used
229
    to encode (e.g. compress or gzip).  The mappings are table
230
    driven.  Encoding suffixes are case sensitive; type suffixes are
231
    first tried case sensitive, then case insensitive.
232
233
    The suffixes .tgz, .taz and .tz (case sensitive!) are all mapped
234
    to ".tar.gz".  (This is table-driven too, using the dictionary
235
    suffix_map).
236
237
    Optional `strict' argument when false adds a bunch of commonly found, but
238
    non-standard types.
239
    """
240
    if not inited:
241
        init()
242
    return guess_type(url, strict)
243
244
2e-06 sec 245
def guess_all_extensions(type, strict=True):
246
    """Guess the extensions for a file based on its MIME type.
247
248
    Return value is a list of strings giving the possible filename
249
    extensions, including the leading dot ('.').  The extension is not
250
    guaranteed to have been associated with any particular data
251
    stream, but would be mapped to the MIME type `type' by
252
    guess_type().  If no extension can be guessed for `type', None
253
    is returned.
254
255
    Optional `strict' argument when false adds a bunch of commonly found,
256
    but non-standard types.
257
    """
258
    if not inited:
259
        init()
260
    return guess_all_extensions(type, strict)
261
3e-06 sec 262
def guess_extension(type, strict=True):
263
    """Guess the extension for a file based on its MIME type.
264
265
    Return value is a string giving a filename extension, including the
266
    leading dot ('.').  The extension is not guaranteed to have been
267
    associated with any particular data stream, but would be mapped to the
268
    MIME type `type' by guess_type().  If no extension can be guessed for
269
    `type', None is returned.
270
271
    Optional `strict' argument when false adds a bunch of commonly found,
272
    but non-standard types.
273
    """
274
    if not inited:
275
        init()
276
    return guess_extension(type, strict)
277
3e-06 sec 278
def add_type(type, ext, strict=True):
279
    """Add a mapping between a type and an extension.
280
281
    When the extension is already known, the new
282
    type will replace the old one. When the type
283
    is already known the extension will be added
284
    to the list of known extensions.
285
286
    If strict is true, information will be added to
287
    list of standard types, else to the list of non-standard
288
    types.
289
    """
290
    if not inited:
291
        init()
292
    return add_type(type, ext, strict)
293
294
9e-06 sec 295
def init(files=None):
296
    global guess_all_extensions, guess_extension, guess_type
297
    global suffix_map, types_map, encodings_map, common_types
298
    global add_type, inited
2e-06 sec 299
    inited = True
2e-06 sec 300
    db = MimeTypes()
4e-06 sec 301
    if files is None:
2e-06 sec 302
        files = knownfiles
7.9e-05 sec 303
    for file in files:
1.8e-05 sec 304
        if os.path.isfile(file):
6e-06 sec 305
            db.readfp(open(file))
3e-06 sec 306
    encodings_map = db.encodings_map
3e-06 sec 307
    suffix_map = db.suffix_map
3e-06 sec 308
    types_map = db.types_map[True]
1.2e-05 sec 309
    guess_all_extensions = db.guess_all_extensions
6e-06 sec 310
    guess_extension = db.guess_extension
5e-06 sec 311
    guess_type = db.guess_type
5e-06 sec 312
    add_type = db.add_type
4e-06 sec 313
    common_types = db.types_map[False]
314
315
2e-06 sec 316
def read_mime_types(file):
317
    try:
318
        f = open(file)
319
    except IOError:
320
        return None
321
    db = MimeTypes()
322
    db.readfp(f, True)
323
    return db.types_map[True]
324
325
7e-06 sec 326
def _default_mime_types():
327
    global suffix_map
328
    global encodings_map
329
    global types_map
330
    global common_types
331
1e-06 sec 332
    suffix_map = {
3e-06 sec 333
        '.tgz': '.tar.gz',
2e-06 sec 334
        '.taz': '.tar.gz',
2e-06 sec 335
        '.tz': '.tar.gz',
2e-06 sec 336
        '.tbz2': '.tar.bz2',
337
        }
338
2e-06 sec 339
    encodings_map = {
2e-06 sec 340
        '.gz': 'gzip',
2e-06 sec 341
        '.Z': 'compress',
1e-06 sec 342
        '.bz2': 'bzip2',
343
        }
344
345
    # Before adding new types, make sure they are either registered with IANA,
346
    # at http://www.isi.edu/in-notes/iana/assignments/media-types
347
    # or extensions, i.e. using the x- prefix
348
349
    # If you add to these, please keep them sorted!
3e-06 sec 350
    types_map = {
3e-06 sec 351
        '.a'      : 'application/octet-stream',
2e-06 sec 352
        '.ai'     : 'application/postscript',
2e-06 sec 353
        '.aif'    : 'audio/x-aiff',
2e-06 sec 354
        '.aifc'   : 'audio/x-aiff',
2e-06 sec 355
        '.aiff'   : 'audio/x-aiff',
2e-06 sec 356
        '.au'     : 'audio/basic',
2e-06 sec 357
        '.avi'    : 'video/x-msvideo',
2e-06 sec 358
        '.bat'    : 'text/plain',
2e-06 sec 359
        '.bcpio'  : 'application/x-bcpio',
1e-06 sec 360
        '.bin'    : 'application/octet-stream',
3e-06 sec 361
        '.bmp'    : 'image/x-ms-bmp',
2e-06 sec 362
        '.c'      : 'text/plain',
363
        # Duplicates :(
2e-06 sec 364
        '.cdf'    : 'application/x-cdf',
2e-06 sec 365
        '.cdf'    : 'application/x-netcdf',
2e-06 sec 366
        '.cpio'   : 'application/x-cpio',
2e-06 sec 367
        '.csh'    : 'application/x-csh',
2e-06 sec 368
        '.css'    : 'text/css',
2e-06 sec 369
        '.dll'    : 'application/octet-stream',
2e-06 sec 370
        '.doc'    : 'application/msword',
2e-06 sec 371
        '.dot'    : 'application/msword',
2e-06 sec 372
        '.dvi'    : 'application/x-dvi',
2e-06 sec 373
        '.eml'    : 'message/rfc822',
2e-06 sec 374
        '.eps'    : 'application/postscript',
2e-06 sec 375
        '.etx'    : 'text/x-setext',
3e-06 sec 376
        '.exe'    : 'application/octet-stream',
2e-06 sec 377
        '.gif'    : 'image/gif',
2e-06 sec 378
        '.gtar'   : 'application/x-gtar',
2e-06 sec 379
        '.h'      : 'text/plain',
2e-06 sec 380
        '.hdf'    : 'application/x-hdf',
2e-06 sec 381
        '.htm'    : 'text/html',
2e-06 sec 382
        '.html'   : 'text/html',
2e-06 sec 383
        '.ief'    : 'image/ief',
3e-06 sec 384
        '.jpe'    : 'image/jpeg',
2e-06 sec 385
        '.jpeg'   : 'image/jpeg',
2e-06 sec 386
        '.jpg'    : 'image/jpeg',
2e-06 sec 387
        '.js'     : 'application/x-javascript',
3e-06 sec 388
        '.ksh'    : 'text/plain',
2e-06 sec 389
        '.latex'  : 'application/x-latex',
3e-06 sec 390
        '.m1v'    : 'video/mpeg',
2e-06 sec 391
        '.man'    : 'application/x-troff-man',
2e-06 sec 392
        '.me'     : 'application/x-troff-me',
2e-06 sec 393
        '.mht'    : 'message/rfc822',
2e-06 sec 394
        '.mhtml'  : 'message/rfc822',
2e-06 sec 395
        '.mif'    : 'application/x-mif',
3e-06 sec 396
        '.mov'    : 'video/quicktime',
2e-06 sec 397
        '.movie'  : 'video/x-sgi-movie',
2e-06 sec 398
        '.mp2'    : 'audio/mpeg',
3e-06 sec 399
        '.mp3'    : 'audio/mpeg',
2e-06 sec 400
        '.mp4'    : 'video/mp4',
2e-06 sec 401
        '.mpa'    : 'video/mpeg',
2e-06 sec 402
        '.mpe'    : 'video/mpeg',
2e-06 sec 403
        '.mpeg'   : 'video/mpeg',
3e-06 sec 404
        '.mpg'    : 'video/mpeg',
2e-06 sec 405
        '.ms'     : 'application/x-troff-ms',
2e-06 sec 406
        '.nc'     : 'application/x-netcdf',
3e-06 sec 407
        '.nws'    : 'message/rfc822',
2e-06 sec 408
        '.o'      : 'application/octet-stream',
3e-06 sec 409
        '.obj'    : 'application/octet-stream',
2e-06 sec 410
        '.oda'    : 'application/oda',
2e-06 sec 411
        '.p12'    : 'application/x-pkcs12',
2e-06 sec 412
        '.p7c'    : 'application/pkcs7-mime',
3e-06 sec 413
        '.pbm'    : 'image/x-portable-bitmap',
2e-06 sec 414
        '.pdf'    : 'application/pdf',
3e-06 sec 415
        '.pfx'    : 'application/x-pkcs12',
2e-06 sec 416
        '.pgm'    : 'image/x-portable-graymap',
3e-06 sec 417
        '.pl'     : 'text/plain',
2e-06 sec 418
        '.png'    : 'image/png',
3e-06 sec 419
        '.pnm'    : 'image/x-portable-anymap',
2e-06 sec 420
        '.pot'    : 'application/vnd.ms-powerpoint',
2e-06 sec 421
        '.ppa'    : 'application/vnd.ms-powerpoint',
2e-06 sec 422
        '.ppm'    : 'image/x-portable-pixmap',
3e-06 sec 423
        '.pps'    : 'application/vnd.ms-powerpoint',
2e-06 sec 424
        '.ppt'    : 'application/vnd.ms-powerpoint',
3e-06 sec 425
        '.ps'     : 'application/postscript',
2e-06 sec 426
        '.pwz'    : 'application/vnd.ms-powerpoint',
3e-06 sec 427
        '.py'     : 'text/x-python',
2e-06 sec 428
        '.pyc'    : 'application/x-python-code',
2e-06 sec 429
        '.pyo'    : 'application/x-python-code',
3e-06 sec 430
        '.qt'     : 'video/quicktime',
2e-06 sec 431
        '.ra'     : 'audio/x-pn-realaudio',
3e-06 sec 432
        '.ram'    : 'application/x-pn-realaudio',
3e-06 sec 433
        '.ras'    : 'image/x-cmu-raster',
2e-06 sec 434
        '.rdf'    : 'application/xml',
3e-06 sec 435
        '.rgb'    : 'image/x-rgb',
2e-06 sec 436
        '.roff'   : 'application/x-troff',
3e-06 sec 437
        '.rtx'    : 'text/richtext',
2e-06 sec 438
        '.sgm'    : 'text/x-sgml',
9e-06 sec 439
        '.sgml'   : 'text/x-sgml',
3e-06 sec 440
        '.sh'     : 'application/x-sh',
3e-06 sec 441
        '.shar'   : 'application/x-shar',
2e-06 sec 442
        '.snd'    : 'audio/basic',
3e-06 sec 443
        '.so'     : 'application/octet-stream',
2e-06 sec 444
        '.src'    : 'application/x-wais-source',
3e-06 sec 445
        '.sv4cpio': 'application/x-sv4cpio',
2e-06 sec 446
        '.sv4crc' : 'application/x-sv4crc',
3e-06 sec 447
        '.swf'    : 'application/x-shockwave-flash',
3e-06 sec 448
        '.t'      : 'application/x-troff',
2e-06 sec 449
        '.tar'    : 'application/x-tar',
3e-06 sec 450
        '.tcl'    : 'application/x-tcl',
3e-06 sec 451
        '.tex'    : 'application/x-tex',
2e-06 sec 452
        '.texi'   : 'application/x-texinfo',
3e-06 sec 453
        '.texinfo': 'application/x-texinfo',
2e-06 sec 454
        '.tif'    : 'image/tiff',
3e-06 sec 455
        '.tiff'   : 'image/tiff',
3e-06 sec 456
        '.tr'     : 'application/x-troff',
2e-06 sec 457
        '.tsv'    : 'text/tab-separated-values',
3e-06 sec 458
        '.txt'    : 'text/plain',
3e-06 sec 459
        '.ustar'  : 'application/x-ustar',
3e-06 sec 460
        '.vcf'    : 'text/x-vcard',
2e-06 sec 461
        '.wav'    : 'audio/x-wav',
3e-06 sec 462
        '.wiz'    : 'application/msword',
2e-06 sec 463
        '.wsdl'   : 'application/xml',
3e-06 sec 464
        '.xbm'    : 'image/x-xbitmap',
3e-06 sec 465
        '.xlb'    : 'application/vnd.ms-excel',
466
        # Duplicates :(
3e-06 sec 467
        '.xls'    : 'application/excel',
2e-06 sec 468
        '.xls'    : 'application/vnd.ms-excel',
3e-06 sec 469
        '.xml'    : 'text/xml',
3e-06 sec 470
        '.xpdl'   : 'application/xml',
2e-06 sec 471
        '.xpm'    : 'image/x-xpixmap',
3e-06 sec 472
        '.xsl'    : 'application/xml',
3e-06 sec 473
        '.xwd'    : 'image/x-xwindowdump',
3e-06 sec 474
        '.zip'    : 'application/zip',
475
        }
476
477
    # These are non-standard types, commonly found in the wild.  They will
478
    # only match if strict=0 flag is given to the API methods.
479
480
    # Please sort these too
5e-06 sec 481
    common_types = {
3e-06 sec 482
        '.jpg' : 'image/jpg',
3e-06 sec 483
        '.mid' : 'audio/midi',
3e-06 sec 484
        '.midi': 'audio/midi',
2e-06 sec 485
        '.pct' : 'image/pict',
3e-06 sec 486
        '.pic' : 'image/pict',
3e-06 sec 487
        '.pict': 'image/pict',
3e-06 sec 488
        '.rtf' : 'application/rtf',
3e-06 sec 489
        '.xul' : 'text/xul'
490
        }
491
492
1e-06 sec 493
_default_mime_types()
494
495
2e-06 sec 496
if __name__ == '__main__':
497
    import sys
498
    import getopt
499
500
    USAGE = """\
501
Usage: mimetypes.py [options] type
502
503
Options:
504
    --help / -h       -- print this message and exit
505
    --lenient / -l    -- additionally search of some common, but non-standard
506
                         types.
507
    --extension / -e  -- guess extension instead of type
508
509
More than one type argument may be given.
510
"""
511
512
    def usage(code, msg=''):
513
        print USAGE
514
        if msg: print msg
515
        sys.exit(code)
516
517
    try:
518
        opts, args = getopt.getopt(sys.argv[1:], 'hle',
519
                                   ['help', 'lenient', 'extension'])
520
    except getopt.error, msg:
521
        usage(1, msg)
522
523
    strict = 1
524
    extension = 0
525
    for opt, arg in opts:
526
        if opt in ('-h', '--help'):
527
            usage(0)
528
        elif opt in ('-l', '--lenient'):
529
            strict = 0
530
        elif opt in ('-e', '--extension'):
531
            extension = 1
532
    for gtype in args:
533
        if extension:
534
            guess = guess_extension(gtype, strict)
535
            if not guess: print "I don't know anything about type", gtype
536
            else: print guess
537
        else:
538
            guess, encoding = guess_type(gtype, strict)
539
            if not guess: print "I don't know anything about type", gtype
540
            else: print 'type:', guess, 'encoding:', encoding
/usr/lib/python2.6/re.py
     time      num code
1
#
2
# Secret Labs' Regular Expression Engine
3
#
4
# re-compatible interface for the sre matching engine
5
#
6
# Copyright (c) 1998-2001 by Secret Labs AB.  All rights reserved.
7
#
8
# This version of the SRE library can be redistributed under CNRI's
9
# Python 1.6 license.  For any other use, please contact Secret Labs
10
# AB (info@pythonware.com).
11
#
12
# Portions of this engine have been developed in cooperation with
13
# CNRI.  Hewlett-Packard provided funding for 1.6 integration and
14
# other compatibility work.
15
#
16
17
r"""Support for regular expressions (RE).
18
19
This module provides regular expression matching operations similar to
20
those found in Perl.  It supports both 8-bit and Unicode strings; both
21
the pattern and the strings being processed can contain null bytes and
22
characters outside the US ASCII range.
23
24
Regular expressions can contain both special and ordinary characters.
25
Most ordinary characters, like "A", "a", or "0", are the simplest
26
regular expressions; they simply match themselves.  You can
27
concatenate ordinary characters, so last matches the string 'last'.
28
29
The special characters are:
30
    "."      Matches any character except a newline.
31
    "^"      Matches the start of the string.
32
    "$"      Matches the end of the string or just before the newline at
33
             the end of the string.
34
    "*"      Matches 0 or more (greedy) repetitions of the preceding RE.
35
             Greedy means that it will match as many repetitions as possible.
36
    "+"      Matches 1 or more (greedy) repetitions of the preceding RE.
37
    "?"      Matches 0 or 1 (greedy) of the preceding RE.
38
    *?,+?,?? Non-greedy versions of the previous three special characters.
39
    {m,n}    Matches from m to n repetitions of the preceding RE.
40
    {m,n}?   Non-greedy version of the above.
41
    "\\"     Either escapes special characters or signals a special sequence.
42
    []       Indicates a set of characters.
43
             A "^" as the first character indicates a complementing set.
44
    "|"      A|B, creates an RE that will match either A or B.
45
    (...)    Matches the RE inside the parentheses.
46
             The contents can be retrieved or matched later in the string.
47
    (?iLmsux) Set the I, L, M, S, U, or X flag for the RE (see below).
48
    (?:...)  Non-grouping version of regular parentheses.
49
    (?P<name>...) The substring matched by the group is accessible by name.
50
    (?P=name)     Matches the text matched earlier by the group named name.
51
    (?#...)  A comment; ignored.
52
    (?=...)  Matches if ... matches next, but doesn't consume the string.
53
    (?!...)  Matches if ... doesn't match next.
54
    (?<=...) Matches if preceded by ... (must be fixed length).
55
    (?<!...) Matches if not preceded by ... (must be fixed length).
56
    (?(id/name)yes|no) Matches yes pattern if the group with id/name matched,
57
                       the (optional) no pattern otherwise.
58
59
The special sequences consist of "\\" and a character from the list
60
below.  If the ordinary character is not on the list, then the
61
resulting RE will match the second character.
62
    \number  Matches the contents of the group of the same number.
63
    \A       Matches only at the start of the string.
64
    \Z       Matches only at the end of the string.
65
    \b       Matches the empty string, but only at the start or end of a word.
66
    \B       Matches the empty string, but not at the start or end of a word.
67
    \d       Matches any decimal digit; equivalent to the set [0-9].
68
    \D       Matches any non-digit character; equivalent to the set [^0-9].
69
    \s       Matches any whitespace character; equivalent to [ \t\n\r\f\v].
70
    \S       Matches any non-whitespace character; equiv. to [^ \t\n\r\f\v].
71
    \w       Matches any alphanumeric character; equivalent to [a-zA-Z0-9_].
72
             With LOCALE, it will match the set [0-9_] plus characters defined
73
             as letters for the current locale.
74
    \W       Matches the complement of \w.
75
    \\       Matches a literal backslash.
76
77
This module exports the following functions:
78
    match    Match a regular expression pattern to the beginning of a string.
79
    search   Search a string for the presence of a pattern.
80
    sub      Substitute occurrences of a pattern found in a string.
81
    subn     Same as sub, but also return the number of substitutions made.
82
    split    Split a string by the occurrences of a pattern.
83
    findall  Find all occurrences of a pattern in a string.
84
    finditer Return an iterator yielding a match object for each match.
85
    compile  Compile a pattern into a RegexObject.
86
    purge    Clear the regular expression cache.
87
    escape   Backslash all non-alphanumerics in a string.
88
89
Some of the functions in this module takes flags as optional parameters:
90
    I  IGNORECASE  Perform case-insensitive matching.
91
    L  LOCALE      Make \w, \W, \b, \B, dependent on the current locale.
92
    M  MULTILINE   "^" matches the beginning of lines (after a newline)
93
                   as well as the string.
94
                   "$" matches the end of lines (before a newline) as well
95
                   as the end of the string.
96
    S  DOTALL      "." matches any character at all, including the newline.
97
    X  VERBOSE     Ignore whitespace and comments for nicer looking RE's.
98
    U  UNICODE     Make \w, \W, \b, \B, dependent on the Unicode locale.
99
100
This module also defines an exception 'error'.
101
102
"""
103
104
import sys
105
import sre_compile
106
import sre_parse
107
108
# public symbols
109
__all__ = [ "match", "search", "sub", "subn", "split", "findall",
110
    "compile", "purge", "template", "escape", "I", "L", "M", "S", "X",
111
    "U", "IGNORECASE", "LOCALE", "MULTILINE", "DOTALL", "VERBOSE",
112
    "UNICODE", "error" ]
113
114
__version__ = "2.2.1"
115
116
# flags
117
I = IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE # ignore case
118
L = LOCALE = sre_compile.SRE_FLAG_LOCALE # assume current 8-bit locale
119
U = UNICODE = sre_compile.SRE_FLAG_UNICODE # assume unicode locale
120
M = MULTILINE = sre_compile.SRE_FLAG_MULTILINE # make anchors look for newline
121
S = DOTALL = sre_compile.SRE_FLAG_DOTALL # make dot match newline
122
X = VERBOSE = sre_compile.SRE_FLAG_VERBOSE # ignore whitespace and comments
123
124
# sre extensions (experimental, don't rely on these)
125
T = TEMPLATE = sre_compile.SRE_FLAG_TEMPLATE # disable backtracking
126
DEBUG = sre_compile.SRE_FLAG_DEBUG # dump pattern after compilation
127
128
# sre exception
129
error = sre_compile.error
130
131
# --------------------------------------------------------------------
132
# public interface
133
134
def match(pattern, string, flags=0):
135
    """Try to apply the pattern at the start of the string, returning
136
    a match object, or None if no match was found."""
137
    return _compile(pattern, flags).match(string)
138
139
def search(pattern, string, flags=0):
140
    """Scan through string looking for a match to the pattern, returning
141
    a match object, or None if no match was found."""
142
    return _compile(pattern, flags).search(string)
143
144
def sub(pattern, repl, string, count=0):
145
    """Return the string obtained by replacing the leftmost
146
    non-overlapping occurrences of the pattern in string by the
147
    replacement repl.  repl can be either a string or a callable;
148
    if a string, backslash escapes in it are processed.  If it is
149
    a callable, it's passed the match object and must return
150
    a replacement string to be used."""
151
    return _compile(pattern, 0).sub(repl, string, count)
152
153
def subn(pattern, repl, string, count=0):
154
    """Return a 2-tuple containing (new_string, number).
155
    new_string is the string obtained by replacing the leftmost
156
    non-overlapping occurrences of the pattern in the source
157
    string by the replacement repl.  number is the number of
158
    substitutions that were made. repl can be either a string or a
159
    callable; if a string, backslash escapes in it are processed.
160
    If it is a callable, it's passed the match object and must
161
    return a replacement string to be used."""
162
    return _compile(pattern, 0).subn(repl, string, count)
163
164
def split(pattern, string, maxsplit=0):
165
    """Split the source string by the occurrences of the pattern,
166
    returning a list containing the resulting substrings."""
167
    return _compile(pattern, 0).split(string, maxsplit)
168
169
def findall(pattern, string, flags=0):
170
    """Return a list of all non-overlapping matches in the string.
171
172
    If one or more groups are present in the pattern, return a
173
    list of groups; this will be a list of tuples if the pattern
174
    has more than one group.
175
176
    Empty matches are included in the result."""
177
    return _compile(pattern, flags).findall(string)
178
179
if sys.hexversion >= 0x02020000:
180
    __all__.append("finditer")
181
    def finditer(pattern, string, flags=0):
182
        """Return an iterator over all non-overlapping matches in the
183
        string.  For each match, the iterator returns a match object.
184
185
        Empty matches are included in the result."""
186
        return _compile(pattern, flags).finditer(string)
187
1e-05 sec 188
def compile(pattern, flags=0):
189
    "Compile a regular expression pattern, returning a pattern object."
2e-06 sec 190
    return _compile(pattern, flags)
191
192
def purge():
193
    "Clear the regular expression cache"
194
    _cache.clear()
195
    _cache_repl.clear()
196
197
def template(pattern, flags=0):
198
    "Compile a template pattern, returning a pattern object"
199
    return _compile(pattern, flags|T)
200
201
_alphanum = {}
202
for c in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890':
203
    _alphanum[c] = 1
204
del c
205
206
def escape(pattern):
207
    "Escape all non-alphanumeric characters in pattern."
208
    s = list(pattern)
209
    alphanum = _alphanum
210
    for i in range(len(pattern)):
211
        c = pattern[i]
212
        if c not in alphanum:
213
            if c == "\000":
214
                s[i] = "\\000"
215
            else:
216
                s[i] = "\\" + c
217
    return pattern[:0].join(s)
218
219
# --------------------------------------------------------------------
220
# internals
221
222
_cache = {}
223
_cache_repl = {}
224
225
_pattern_type = type(sre_compile.compile("", 0))
226
227
_MAXCACHE = 100
228
7e-06 sec 229
def _compile(*key):
230
    # internal: compile pattern
2e-06 sec 231
    cachekey = (type(key[0]),) + key
8e-06 sec 232
    p = _cache.get(cachekey)
7e-06 sec 233
    if p is not None:
234
        return p
3e-06 sec 235
    pattern, flags = key
2e-06 sec 236
    if isinstance(pattern, _pattern_type):
237
        if flags:
238
            raise ValueError('Cannot process flags argument with a compiled pattern')
239
        return pattern
8e-06 sec 240
    if not sre_compile.isstring(pattern):
241
        raise TypeError, "first argument must be string or compiled pattern"
3e-06 sec 242
    try:
1e-06 sec 243
        p = sre_compile.compile(pattern, flags)
244
    except error, v:
245
        raise error, v # invalid expression
6e-06 sec 246
    if len(_cache) >= _MAXCACHE:
247
        _cache.clear()
4e-06 sec 248
    _cache[cachekey] = p
3e-06 sec 249
    return p
250
251
def _compile_repl(*key):
252
    # internal: compile replacement pattern
253
    p = _cache_repl.get(key)
254
    if p is not None:
255
        return p
256
    repl, pattern = key
257
    try:
258
        p = sre_parse.parse_template(repl, pattern)
259
    except error, v:
260
        raise error, v # invalid expression
261
    if len(_cache_repl) >= _MAXCACHE:
262
        _cache_repl.clear()
263
    _cache_repl[key] = p
264
    return p
265
266
def _expand(pattern, match, template):
267
    # internal: match.expand implementation hook
268
    template = sre_parse.parse_template(template, pattern)
269
    return sre_parse.expand_template(template, match)
270
271
def _subx(pattern, template):
272
    # internal: pattern.sub/subn implementation helper
273
    template = _compile_repl(template, pattern)
274
    if not template[0] and len(template[1]) == 1:
275
        # literal replacement
276
        return template[1][0]
277
    def filter(match, template=template):
278
        return sre_parse.expand_template(template, match)
279
    return filter
280
281
# register myself for pickling
282
283
import copy_reg
284
285
def _pickle(p):
286
    return _compile, (p.pattern, p.flags)
287
288
copy_reg.pickle(_pattern_type, _pickle, _compile)
289
290
# --------------------------------------------------------------------
291
# experimental stuff (see python-dev discussions for details)
292
293
class Scanner:
294
    def __init__(self, lexicon, flags=0):
295
        from sre_constants import BRANCH, SUBPATTERN
296
        self.lexicon = lexicon
297
        # combine phrases into a compound pattern
298
        p = []
299
        s = sre_parse.Pattern()
300
        s.flags = flags
301
        for phrase, action in lexicon:
302
            p.append(sre_parse.SubPattern(s, [
303
                (SUBPATTERN, (len(p)+1, sre_parse.parse(phrase, flags))),
304
                ]))
305
        s.groups = len(p)+1
306
        p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
307
        self.scanner = sre_compile.compile(p)
308
    def scan(self, string):
309
        result = []
310
        append = result.append
311
        match = self.scanner.scanner(string).match
312
        i = 0
313
        while 1:
314
            m = match()
315
            if not m:
316
                break
317
            j = m.end()
318
            if i == j:
319
                break
320
            action = self.lexicon[m.lastindex-1][1]
321
            if hasattr(action, '__call__'):
322
                self.match = m
323
                action = action(self, m.group())
324
            if action is not None:
325
                append(action)
326
            i = j
327
        return result, string[i:]
/usr/lib/python2.6/shutil.py
     time      num code
1
"""Utility functions for copying files and directory trees.
2
3
XXX The functions here don't copy the resource fork or other metadata on Mac.
4
0.000637 sec 5
"""
6
2e-06 sec 7
import os
9e-06 sec 8
import sys
4e-06 sec 9
import stat
5e-06 sec 10
from os.path import abspath
1.4e-05 sec 11
import fnmatch
12
5e-06 sec 13
__all__ = ["copyfileobj","copyfile","copymode","copystat","copy","copy2",
2e-06 sec 14
           "copytree","move","rmtree","Error"]
15
1e-05 sec 16
class Error(EnvironmentError):
2e-06 sec 17
    pass
18
0.000644 sec 19
try:
2e-06 sec 20
    WindowsError
1.7e-05 sec 21
except NameError:
8e-06 sec 22
    WindowsError = None
23
2e-06 sec 24
def copyfileobj(fsrc, fdst, length=16*1024):
25
    """copy data from file-like object fsrc to file-like object fdst"""
26
    while 1:
27
        buf = fsrc.read(length)
28
        if not buf:
29
            break
30
        fdst.write(buf)
31
3e-06 sec 32
def _samefile(src, dst):
33
    # Macintosh, Unix.
34
    if hasattr(os.path,'samefile'):
35
        try:
36
            return os.path.samefile(src, dst)
37
        except OSError:
38
            return False
39
40
    # All other platforms: check for same pathname.
41
    return (os.path.normcase(os.path.abspath(src)) ==
42
            os.path.normcase(os.path.abspath(dst)))
43
2e-06 sec 44
def copyfile(src, dst):
45
    """Copy data from src to dst"""
46
    if _samefile(src, dst):
47
        raise Error, "`%s` and `%s` are the same file" % (src, dst)
48
49
    fsrc = None
50
    fdst = None
51
    try:
52
        fsrc = open(src, 'rb')
53
        fdst = open(dst, 'wb')
54
        copyfileobj(fsrc, fdst)
55
    finally:
56
        if fdst:
57
            fdst.close()
58
        if fsrc:
59
            fsrc.close()
60
2e-06 sec 61
def copymode(src, dst):
62
    """Copy mode bits from src to dst"""
63
    if hasattr(os, 'chmod'):
64
        st = os.stat(src)
65
        mode = stat.S_IMODE(st.st_mode)
66
        os.chmod(dst, mode)
67
3e-06 sec 68
def copystat(src, dst):
69
    """Copy all stat info (mode bits, atime, mtime, flags) from src to dst"""
70
    st = os.stat(src)
71
    mode = stat.S_IMODE(st.st_mode)
72
    if hasattr(os, 'utime'):
73
        os.utime(dst, (st.st_atime, st.st_mtime))
74
    if hasattr(os, 'chmod'):
75
        os.chmod(dst, mode)
76
    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
77
        os.chflags(dst, st.st_flags)
78
79
2e-06 sec 80
def copy(src, dst):
81
    """Copy data and mode bits ("cp src dst").
82
83
    The destination may be a directory.
84
85
    """
86
    if os.path.isdir(dst):
87
        dst = os.path.join(dst, os.path.basename(src))
88
    copyfile(src, dst)
89
    copymode(src, dst)
90
2e-06 sec 91
def copy2(src, dst):
92
    """Copy data and all stat info ("cp -p src dst").
93
94
    The destination may be a directory.
95
96
    """
97
    if os.path.isdir(dst):
98
        dst = os.path.join(dst, os.path.basename(src))
99
    copyfile(src, dst)
100
    copystat(src, dst)
101
2e-06 sec 102
def ignore_patterns(*patterns):
103
    """Function that can be used as copytree() ignore parameter.
104
105
    Patterns is a sequence of glob-style patterns
106
    that are used to exclude files"""
107
    def _ignore_patterns(path, names):
108
        ignored_names = []
109
        for pattern in patterns:
110
            ignored_names.extend(fnmatch.filter(names, pattern))
111
        return set(ignored_names)
112
    return _ignore_patterns
113
2e-06 sec 114
def copytree(src, dst, symlinks=False, ignore=None):
115
    """Recursively copy a directory tree using copy2().
116
117
    The destination directory must not already exist.
118
    If exception(s) occur, an Error is raised with a list of reasons.
119
120
    If the optional symlinks flag is true, symbolic links in the
121
    source tree result in symbolic links in the destination tree; if
122
    it is false, the contents of the files pointed to by symbolic
123
    links are copied.
124
125
    The optional ignore argument is a callable. If given, it
126
    is called with the `src` parameter, which is the directory
127
    being visited by copytree(), and `names` which is the list of
128
    `src` contents, as returned by os.listdir():
129
130
        callable(src, names) -> ignored_names
131
132
    Since copytree() is called recursively, the callable will be
133
    called once for each directory that is copied. It returns a
134
    list of names relative to the `src` directory that should
135
    not be copied.
136
137
    XXX Consider this example code rather than the ultimate tool.
138
139
    """
140
    names = os.listdir(src)
141
    if ignore is not None:
142
        ignored_names = ignore(src, names)
143
    else:
144
        ignored_names = set()
145
146
    os.makedirs(dst)
147
    errors = []
148
    for name in names:
149
        if name in ignored_names:
150
            continue
151
        srcname = os.path.join(src, name)
152
        dstname = os.path.join(dst, name)
153
        try:
154
            if symlinks and os.path.islink(srcname):
155
                linkto = os.readlink(srcname)
156
                os.symlink(linkto, dstname)
157
            elif os.path.isdir(srcname):
158
                copytree(srcname, dstname, symlinks, ignore)
159
            else:
160
                copy2(srcname, dstname)
161
            # XXX What about devices, sockets etc.?
162
        except (IOError, os.error), why:
163
            errors.append((srcname, dstname, str(why)))
164
        # catch the Error from the recursive copytree so that we can
165
        # continue with other files
166
        except Error, err:
167
            errors.extend(err.args[0])
168
    try:
169
        copystat(src, dst)
170
    except OSError, why:
171
        if WindowsError is not None and isinstance(why, WindowsError):
172
            # Copying file access times may fail on Windows
173
            pass
174
        else:
175
            errors.extend((src, dst, str(why)))
176
    if errors:
177
        raise Error, errors
178
8e-06 sec 179
def rmtree(path, ignore_errors=False, onerror=None):
180
    """Recursively delete a directory tree.
181
182
    If ignore_errors is set, errors are ignored; otherwise, if onerror
183
    is set, it is called to handle the error with arguments (func,
184
    path, exc_info) where func is os.listdir, os.remove, or os.rmdir;
185
    path is the argument to that function that caused it to fail; and
186
    exc_info is a tuple returned by sys.exc_info().  If ignore_errors
187
    is false and onerror is None, an exception is raised.
188
189
    """
190
    if ignore_errors:
191
        def onerror(*args):
192
            pass
193
    elif onerror is None:
194
        def onerror(*args):
195
            raise
196
    try:
197
        if os.path.islink(path):
198
            # symlinks to directories are forbidden, see bug #1669
199
            raise OSError("Cannot call rmtree on a symbolic link")
200
    except OSError:
201
        onerror(os.path.islink, path, sys.exc_info())
202
        # can't continue even if onerror hook returns
203
        return
204
    names = []
205
    try:
206
        names = os.listdir(path)
207
    except os.error, err:
208
        onerror(os.listdir, path, sys.exc_info())
209
    for name in names:
210
        fullname = os.path.join(path, name)
211
        try:
212
            mode = os.lstat(fullname).st_mode
213
        except os.error:
214
            mode = 0
215
        if stat.S_ISDIR(mode):
216
            rmtree(fullname, ignore_errors, onerror)
217
        else:
218
            try:
219
                os.remove(fullname)
220
            except os.error, err:
221
                onerror(os.remove, fullname, sys.exc_info())
222
    try:
223
        os.rmdir(path)
224
    except os.error:
225
        onerror(os.rmdir, path, sys.exc_info())
226
227
3e-06 sec 228
def _basename(path):
229
    # A basename() variant which first strips the trailing slash, if present.
230
    # Thus we always get the last component of the path, even for directories.
231
    return os.path.basename(path.rstrip(os.path.sep))
232
2e-06 sec 233
def move(src, dst):
234
    """Recursively move a file or directory to another location. This is
235
    similar to the Unix "mv" command.
236
237
    If the destination is a directory or a symlink to a directory, the source
238
    is moved inside the directory. The destination path must not already
239
    exist.
240
241
    If the destination already exists but is not a directory, it may be
242
    overwritten depending on os.rename() semantics.
243
244
    If the destination is on our current filesystem, then rename() is used.
245
    Otherwise, src is copied to the destination and then removed.
246
    A lot more could be done here...  A look at a mv.c shows a lot of
247
    the issues this implementation glosses over.
248
249
    """
250
    real_dst = dst
251
    if os.path.isdir(dst):
252
        real_dst = os.path.join(dst, _basename(src))
253
        if os.path.exists(real_dst):
254
            raise Error, "Destination path '%s' already exists" % real_dst
255
    try:
256
        os.rename(src, real_dst)
257
    except OSError:
258
        if os.path.isdir(src):
259
            if destinsrc(src, dst):
260
                raise Error, "Cannot move a directory '%s' into itself '%s'." % (src, dst)
261
            copytree(src, real_dst, symlinks=True)
262
            rmtree(src)
263
        else:
264
            copy2(src, real_dst)
265
            os.unlink(src)
266
3e-06 sec 267
def destinsrc(src, dst):
268
    src = abspath(src)
269
    dst = abspath(dst)
270
    if not src.endswith(os.path.sep):
271
        src += os.path.sep
272
    if not dst.endswith(os.path.sep):
273
        dst += os.path.sep
274
    return dst.startswith(src)
/usr/lib/python2.6/socket.py
     time      num code
1
# Wrapper module for _socket, providing some additional facilities
2
# implemented in Python.
3
4
"""\
5
This module provides socket operations and some related functions.
6
On Unix, it supports IP (Internet Protocol) and Unix domain sockets.
7
On other systems, it only supports IP. Functions specific for a
8
socket are available as methods of the socket object.
9
10
Functions:
11
12
socket() -- create a new socket object
13
socketpair() -- create a pair of new socket objects [*]
14
fromfd() -- create a socket object from an open file descriptor [*]
15
gethostname() -- return the current hostname
16
gethostbyname() -- map a hostname to its IP number
17
gethostbyaddr() -- map an IP number or hostname to DNS info
18
getservbyname() -- map a service name and a protocol name to a port number
19
getprotobyname() -- mape a protocol name (e.g. 'tcp') to a number
20
ntohs(), ntohl() -- convert 16, 32 bit int from network to host byte order
21
htons(), htonl() -- convert 16, 32 bit int from host to network byte order
22
inet_aton() -- convert IP addr string (123.45.67.89) to 32-bit packed format
23
inet_ntoa() -- convert 32-bit packed format IP to string (123.45.67.89)
24
ssl() -- secure socket layer support (only available if configured)
25
socket.getdefaulttimeout() -- get the default timeout value
26
socket.setdefaulttimeout() -- set the default timeout value
27
create_connection() -- connects to an address, with an optional timeout
28
29
 [*] not available on all platforms!
30
31
Special objects:
32
33
SocketType -- type object for socket objects
34
error -- exception raised for I/O errors
35
has_ipv6 -- boolean value indicating if IPv6 is supported
36
37
Integer constants:
38
39
AF_INET, AF_UNIX -- socket domains (first argument to socket() call)
40
SOCK_STREAM, SOCK_DGRAM, SOCK_RAW -- socket types (second argument)
41
42
Many other constants may be defined; these may be used in calls to
43
the setsockopt() and getsockopt() methods.
44
"""
45
46
import _socket
47
from _socket import *
48
49
try:
50
    import _ssl
51
except ImportError:
52
    # no SSL support
53
    pass
54
else:
55
    def ssl(sock, keyfile=None, certfile=None):
56
        # we do an internal import here because the ssl
57
        # module imports the socket module
58
        import ssl as _realssl
59
        warnings.warn("socket.ssl() is deprecated.  Use ssl.wrap_socket() instead.",
60
                      DeprecationWarning, stacklevel=2)
61
        return _realssl.sslwrap_simple(sock, keyfile, certfile)
62
63
    # we need to import the same constants we used to...
64
    from _ssl import SSLError as sslerror
65
    from _ssl import \
66
         RAND_add, \
67
         RAND_egd, \
68
         RAND_status, \
69
         SSL_ERROR_ZERO_RETURN, \
70
         SSL_ERROR_WANT_READ, \
71
         SSL_ERROR_WANT_WRITE, \
72
         SSL_ERROR_WANT_X509_LOOKUP, \
73
         SSL_ERROR_SYSCALL, \
74
         SSL_ERROR_SSL, \
75
         SSL_ERROR_WANT_CONNECT, \
76
         SSL_ERROR_EOF, \
77
         SSL_ERROR_INVALID_ERROR_CODE
78
79
import os, sys, warnings
80
81
try:
82
    from cStringIO import StringIO
83
except ImportError:
84
    from StringIO import StringIO
85
86
try:
87
    from errno import EBADF
88
except ImportError:
89
    EBADF = 9
90
91
__all__ = ["getfqdn", "create_connection"]
92
__all__.extend(os._get_exports_list(_socket))
93
94
95
_realsocket = socket
96
97
# WSA error codes
98
if sys.platform.lower().startswith("win"):
99
    errorTab = {}
100
    errorTab[10004] = "The operation was interrupted."
101
    errorTab[10009] = "A bad file handle was passed."
102
    errorTab[10013] = "Permission denied."
103
    errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT
104
    errorTab[10022] = "An invalid operation was attempted."
105
    errorTab[10035] = "The socket operation would block"
106
    errorTab[10036] = "A blocking operation is already in progress."
107
    errorTab[10048] = "The network address is in use."
108
    errorTab[10054] = "The connection has been reset."
109
    errorTab[10058] = "The network has been shut down."
110
    errorTab[10060] = "The operation timed out."
111
    errorTab[10061] = "Connection refused."
112
    errorTab[10063] = "The name is too long."
113
    errorTab[10064] = "The host is down."
114
    errorTab[10065] = "The host is unreachable."
115
    __all__.append("errorTab")
116
117
118
119
def getfqdn(name=''):
120
    """Get fully qualified domain name from name.
121
122
    An empty argument is interpreted as meaning the local host.
123
124
    First the hostname returned by gethostbyaddr() is checked, then
125
    possibly existing aliases. In case no FQDN is available, hostname
126
    from gethostname() is returned.
127
    """
128
    name = name.strip()
129
    if not name or name == '0.0.0.0':
130
        name = gethostname()
131
    try:
132
        hostname, aliases, ipaddrs = gethostbyaddr(name)
133
    except error:
134
        pass
135
    else:
136
        aliases.insert(0, hostname)
137
        for name in aliases:
138
            if '.' in name:
139
                break
140
        else:
141
            name = hostname
142
    return name
143
144
145
_socketmethods = (
146
    'bind', 'connect', 'connect_ex', 'fileno', 'listen',
147
    'getpeername', 'getsockname', 'getsockopt', 'setsockopt',
148
    'sendall', 'setblocking',
149
    'settimeout', 'gettimeout', 'shutdown')
150
151
if os.name == "nt":
152
    _socketmethods = _socketmethods + ('ioctl',)
153
154
if sys.platform == "riscos":
155
    _socketmethods = _socketmethods + ('sleeptaskw',)
156
157
# All the method names that must be delegated to either the real socket
158
# object or the _closedsocket object.
159
_delegate_methods = ("recv", "recvfrom", "recv_into", "recvfrom_into",
160
                     "send", "sendto")
161
162
class _closedsocket(object):
163
    __slots__ = []
164
    def _dummy(*args):
165
        raise error(EBADF, 'Bad file descriptor')
166
    # All _delegate_methods must also be initialized here.
167
    send = recv = recv_into = sendto = recvfrom = recvfrom_into = _dummy
168
    __getattr__ = _dummy
169
170
# Wrapper around platform socket objects. This implements
171
# a platform-independent dup() functionality. The
172
# implementation currently relies on reference counting
173
# to close the underlying socket object.
174
class _socketobject(object):
175
176
    __doc__ = _realsocket.__doc__
177
178
    __slots__ = ["_sock", "__weakref__"] + list(_delegate_methods)
179
6.1e-05 sec 180
    def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, _sock=None):
6e-06 sec 181
        if _sock is None:
5e-06 sec 182
            _sock = _realsocket(family, type, proto)
4.3e-05 sec 183
        self._sock = _sock
0.000115 sec 184
        for method in _delegate_methods:
4e-05 sec 185
            setattr(self, method, getattr(_sock, method))
186
1.8e-05 sec 187
    def close(self):
4e-06 sec 188
        self._sock = _closedsocket()
8e-06 sec 189
        dummy = self._sock._dummy
0.000109 sec 190
        for method in _delegate_methods:
2.1e-05 sec 191
            setattr(self, method, dummy)
192
    close.__doc__ = _realsocket.close.__doc__
193
7.3559 sec 194
    def accept(self):
1.6e-05 sec 195
        sock, addr = self._sock.accept()
0.049989 sec 196
        return _socketobject(_sock=sock), addr
197
    accept.__doc__ = _realsocket.accept.__doc__
198
199
    def dup(self):
200
        """dup() -> socket object
201
202
        Return a new socket object connected to the same system resource."""
203
        return _socketobject(_sock=self._sock)
204
205
    def makefile(self, mode='r', bufsize=-1):
206
        """makefile([mode[, bufsize]]) -> file object
207
208
        Return a regular file object corresponding to the socket.  The mode
209
        and bufsize arguments are as for the built-in open() function."""
210
        return _fileobject(self._sock, mode, bufsize)
211
212
    family = property(lambda self: self._sock.family, doc="the socket family")
213
    type = property(lambda self: self._sock.type, doc="the socket type")
214
    proto = property(lambda self: self._sock.proto, doc="the socket protocol")
215
216
    _s = ("def %s(self, *args): return self._sock.%s(*args)\n\n"
217
          "%s.__doc__ = _realsocket.%s.__doc__\n")
218
    for _m in _socketmethods:
219
        exec _s % (_m, _m, _m, _m)
220
    del _m, _s
221
222
socket = SocketType = _socketobject
223
224
class _fileobject(object):
225
    """Faux file object attached to a socket object."""
226
227
    default_bufsize = 8192
228
    name = "<socket>"
229
230
    __slots__ = ["mode", "bufsize", "softspace",
231
                 # "closed" is a property, see below
232
                 "_sock", "_rbufsize", "_wbufsize", "_rbuf", "_wbuf",
233
                 "_close"]
234
3.4e-05 sec 235
    def __init__(self, sock, mode='rb', bufsize=-1, close=False):
5e-06 sec 236
        self._sock = sock
1.1e-05 sec 237
        self.mode = mode # Not actually used in this version
8e-06 sec 238
        if bufsize < 0:
8e-06 sec 239
            bufsize = self.default_bufsize
8e-06 sec 240
        self.bufsize = bufsize
6e-06 sec 241
        self.softspace = False
242
        # _rbufsize is the suggested recv buffer size.  It is *strictly*
243
        # obeyed within readline() for recv calls.  If it is larger than
244
        # default_bufsize it will be used for recv calls within read().
8e-06 sec 245
        if bufsize == 0:
246
            self._rbufsize = 1
9e-06 sec 247
        elif bufsize == 1:
248
            self._rbufsize = self.default_bufsize
249
        else:
6e-06 sec 250
            self._rbufsize = bufsize
8e-06 sec 251
        self._wbufsize = bufsize
252
        # We use StringIO for the read buffer to avoid holding a list
253
        # of variously sized string objects which have been known to
254
        # fragment the heap due to how they are malloc()ed and often
255
        # realloc()ed down much smaller than their original allocation.
7e-06 sec 256
        self._rbuf = StringIO()
2.4e-05 sec 257
        self._wbuf = [] # A list of strings
8e-06 sec 258
        self._close = close
259
2.4e-05 sec 260
    def _getclosed(self):
2e-06 sec 261
        return self._sock is None
262
    closed = property(_getclosed, doc="True if the file is closed")
263
1.9e-05 sec 264
    def close(self):
5e-06 sec 265
        try:
6e-06 sec 266
            if self._sock:
2e-06 sec 267
                self.flush()
268
        finally:
6e-06 sec 269
            if self._close:
270
                self._sock.close()
7e-06 sec 271
            self._sock = None
272
4.1e-05 sec 273
    def __del__(self):
7e-06 sec 274
        try:
5e-06 sec 275
            self.close()
276
        except:
277
            # close() may fail if __init__ didn't complete
278
            pass
279
280
    def flush(self):
281
        if self._wbuf:
282
            buffer = "".join(self._wbuf)
283
            self._wbuf = []
284
            self._sock.sendall(buffer)
285
286
    def fileno(self):
287
        return self._sock.fileno()
288
289
    def write(self, data):
290
        data = str(data) # XXX Should really reject non-string non-buffers
291
        if not data:
292
            return
293
        self._wbuf.append(data)
294
        if (self._wbufsize == 0 or
295
            self._wbufsize == 1 and '\n' in data or
296
            self._get_wbuf_len() >= self._wbufsize):
297
            self.flush()
298
299
    def writelines(self, list):
300
        # XXX We could do better here for very long lists
301
        # XXX Should really reject non-string non-buffers
302
        self._wbuf.extend(filter(None, map(str, list)))
303
        if (self._wbufsize <= 1 or
304
            self._get_wbuf_len() >= self._wbufsize):
305
            self.flush()
306
307
    def _get_wbuf_len(self):
308
        buf_len = 0
309
        for x in self._wbuf:
310
            buf_len += len(x)
311
        return buf_len
312
313
    def read(self, size=-1):
314
        # Use max, disallow tiny reads in a loop as they are very inefficient.
315
        # We never leave read() with any leftover data from a new recv() call
316
        # in our internal buffer.
317
        rbufsize = max(self._rbufsize, self.default_bufsize)
318
        # Our use of StringIO rather than lists of string objects returned by
319
        # recv() minimizes memory usage and fragmentation that occurs when
320
        # rbufsize is large compared to the typical return value of recv().
321
        buf = self._rbuf
322
        buf.seek(0, 2)  # seek end
323
        if size < 0:
324
            # Read until EOF
325
            self._rbuf = StringIO()  # reset _rbuf.  we consume it via buf.
326
            while True:
327
                data = self._sock.recv(rbufsize)
328
                if not data:
329
                    break
330
                buf.write(data)
331
            return buf.getvalue()
332
        else:
333
            # Read until size bytes or EOF seen, whichever comes first
334
            buf_len = buf.tell()
335
            if buf_len >= size:
336
                # Already have size bytes in our buffer?  Extract and return.
337
                buf.seek(0)
338
                rv = buf.read(size)
339
                self._rbuf = StringIO()
340
                self._rbuf.write(buf.read())
341
                return rv
342
343
            self._rbuf = StringIO()  # reset _rbuf.  we consume it via buf.
344
            while True:
345
                left = size - buf_len
346
                # recv() will malloc the amount of memory given as its
347
                # parameter even though it often returns much less data
348
                # than that.  The returned data string is short lived
349
                # as we copy it into a StringIO and free it.  This avoids
350
                # fragmentation issues on many platforms.
351
                data = self._sock.recv(left)
352
                if not data:
353
                    break
354
                n = len(data)
355
                if n == size and not buf_len:
356
                    # Shortcut.  Avoid buffer data copies when:
357
                    # - We have no data in our buffer.
358
                    # AND
359
                    # - Our call to recv returned exactly the
360
                    #   number of bytes we were asked to read.
361
                    return data
362
                if n == left:
363
                    buf.write(data)
364
                    del data  # explicit free
365
                    break
366
                assert n <= left, "recv(%d) returned %d bytes" % (left, n)
367
                buf.write(data)
368
                buf_len += n
369
                del data  # explicit free
370
                #assert buf_len == buf.tell()
371
            return buf.getvalue()
372
373
    def readline(self, size=-1):
374
        buf = self._rbuf
375
        buf.seek(0, 2)  # seek end
376
        if buf.tell() > 0:
377
            # check if we already have it in our buffer
378
            buf.seek(0)
379
            bline = buf.readline(size)
380
            if bline.endswith('\n') or len(bline) == size:
381
                self._rbuf = StringIO()
382
                self._rbuf.write(buf.read())
383
                return bline
384
            del bline
385
        if size < 0:
386
            # Read until \n or EOF, whichever comes first
387
            if self._rbufsize <= 1:
388
                # Speed up unbuffered case
389
                buf.seek(0)
390
                buffers = [buf.read()]
391
                self._rbuf = StringIO()  # reset _rbuf.  we consume it via buf.
392
                data = None
393
                recv = self._sock.recv
394
                while data != "\n":
395
                    data = recv(1)
396
                    if not data:
397
                        break
398
                    buffers.append(data)
399
                return "".join(buffers)
400
401
            buf.seek(0, 2)  # seek end
402
            self._rbuf = StringIO()  # reset _rbuf.  we consume it via buf.
403
            while True:
404
                data = self._sock.recv(self._rbufsize)
405
                if not data:
406
                    break
407
                nl = data.find('\n')
408
                if nl >= 0:
409
                    nl += 1
410
                    buf.write(data[:nl])
411
                    self._rbuf.write(data[nl:])
412
                    del data
413
                    break
414
                buf.write(data)
415
            return buf.getvalue()
416
        else:
417
            # Read until size bytes or \n or EOF seen, whichever comes first
418
            buf.seek(0, 2)  # seek end
419
            buf_len = buf.tell()
420
            if buf_len >= size:
421
                buf.seek(0)
422
                rv = buf.read(size)
423
                self._rbuf = StringIO()
424
                self._rbuf.write(buf.read())
425
                return rv
426
            self._rbuf = StringIO()  # reset _rbuf.  we consume it via buf.
427
            while True:
428
                data = self._sock.recv(self._rbufsize)
429
                if not data:
430
                    break
431
                left = size - buf_len
432
                # did we just receive a newline?
433
                nl = data.find('\n', 0, left)
434
                if nl >= 0:
435
                    nl += 1
436
                    # save the excess data to _rbuf
437
                    self._rbuf.write(data[nl:])
438
                    if buf_len:
439
                        buf.write(data[:nl])
440
                        break
441
                    else:
442
                        # Shortcut.  Avoid data copy through buf when returning
443
                        # a substring of our first recv().
444
                        return data[:nl]
445
                n = len(data)
446
                if n == size and not buf_len:
447
                    # Shortcut.  Avoid data copy through buf when
448
                    # returning exactly all of our first recv().
449
                    return data
450
                if n >= left:
451
                    buf.write(data[:left])
452
                    self._rbuf.write(data[left:])
453
                    break
454
                buf.write(data)
455
                buf_len += n
456
                #assert buf_len == buf.tell()
457
            return buf.getvalue()
458
459
    def readlines(self, sizehint=0):
460
        total = 0
461
        list = []
462
        while True:
463
            line = self.readline()
464
            if not line:
465
                break
466
            list.append(line)
467
            total += len(line)
468
            if sizehint and total >= sizehint:
469
                break
470
        return list
471
472
    # Iterator protocols
473
474
    def __iter__(self):
475
        return self
476
477
    def next(self):
478
        line = self.readline()
479
        if not line:
480
            raise StopIteration
481
        return line
482
483
_GLOBAL_DEFAULT_TIMEOUT = object()
484
485
def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT):
486
    """Connect to *address* and return the socket object.
487
488
    Convenience function.  Connect to *address* (a 2-tuple ``(host,
489
    port)``) and return the socket object.  Passing the optional
490
    *timeout* parameter will set the timeout on the socket instance
491
    before attempting to connect.  If no *timeout* is supplied, the
492
    global default timeout setting returned by :func:`getdefaulttimeout`
493
    is used.
494
    """
495
496
    msg = "getaddrinfo returns an empty list"
497
    host, port = address
498
    for res in getaddrinfo(host, port, 0, SOCK_STREAM):
499
        af, socktype, proto, canonname, sa = res
500
        sock = None
501
        try:
502
            sock = socket(af, socktype, proto)
503
            if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
504
                sock.settimeout(timeout)
505
            sock.connect(sa)
506
            return sock
507
508
        except error, msg:
509
            if sock is not None:
510
                sock.close()
511
512
    raise error, msg
/usr/lib/python2.6/sre_compile.py
     time      num code
1
#
2
# Secret Labs' Regular Expression Engine
3
#
4
# convert template to internal format
5
#
6
# Copyright (c) 1997-2001 by Secret Labs AB.  All rights reserved.
7
#
8
# See the sre.py file for information on usage and redistribution.
9
#
10
11
"""Internal support module for sre"""
12
13
import _sre, sys
14
import sre_parse
15
from sre_constants import *
16
17
assert _sre.MAGIC == MAGIC, "SRE module mismatch"
18
19
if _sre.CODESIZE == 2:
20
    MAXCODE = 65535
21
else:
22
    MAXCODE = 0xFFFFFFFFL
23
24
def _identityfunction(x):
25
    return x
26
27
def set(seq):
28
    s = {}
29
    for elem in seq:
30
        s[elem] = 1
31
    return s
32
33
_LITERAL_CODES = set([LITERAL, NOT_LITERAL])
34
_REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT])
35
_SUCCESS_CODES = set([SUCCESS, FAILURE])
36
_ASSERT_CODES = set([ASSERT, ASSERT_NOT])
37
9e-06 sec 38
def _compile(code, pattern, flags):
39
    # internal: compile a (sub)pattern
2e-06 sec 40
    emit = code.append
2e-06 sec 41
    _len = len
2e-06 sec 42
    LITERAL_CODES = _LITERAL_CODES
3e-06 sec 43
    REPEATING_CODES = _REPEATING_CODES
2e-06 sec 44
    SUCCESS_CODES = _SUCCESS_CODES
2e-06 sec 45
    ASSERT_CODES = _ASSERT_CODES
2.3e-05 sec 46
    for op, av in pattern:
6e-06 sec 47
        if op in LITERAL_CODES:
6e-06 sec 48
            if flags & SRE_FLAG_IGNORECASE:
7e-06 sec 49
                emit(OPCODES[OP_IGNORE[op]])
8e-06 sec 50
                emit(_sre.getlower(av, flags))
51
            else:
52
                emit(OPCODES[op])
53
                emit(av)
54
        elif op is IN:
55
            if flags & SRE_FLAG_IGNORECASE:
56
                emit(OPCODES[OP_IGNORE[op]])
57
                def fixup(literal, flags=flags):
58
                    return _sre.getlower(literal, flags)
59
            else:
60
                emit(OPCODES[op])
61
                fixup = _identityfunction
62
            skip = _len(code); emit(0)
63
            _compile_charset(av, flags, code, fixup)
64
            code[skip] = _len(code) - skip
65
        elif op is ANY:
66
            if flags & SRE_FLAG_DOTALL:
67
                emit(OPCODES[ANY_ALL])
68
            else:
69
                emit(OPCODES[ANY])
70
        elif op in REPEATING_CODES:
71
            if flags & SRE_FLAG_TEMPLATE:
72
                raise error, "internal: unsupported template operator"
73
                emit(OPCODES[REPEAT])
74
                skip = _len(code); emit(0)
75
                emit(av[0])
76
                emit(av[1])
77
                _compile(code, av[2], flags)
78
                emit(OPCODES[SUCCESS])
79
                code[skip] = _len(code) - skip
80
            elif _simple(av) and op is not REPEAT:
81
                if op is MAX_REPEAT:
82
                    emit(OPCODES[REPEAT_ONE])
83
                else:
84
                    emit(OPCODES[MIN_REPEAT_ONE])
85
                skip = _len(code); emit(0)
86
                emit(av[0])
87
                emit(av[1])
88
                _compile(code, av[2], flags)
89
                emit(OPCODES[SUCCESS])
90
                code[skip] = _len(code) - skip
91
            else:
92
                emit(OPCODES[REPEAT])
93
                skip = _len(code); emit(0)
94
                emit(av[0])
95
                emit(av[1])
96
                _compile(code, av[2], flags)
97
                code[skip] = _len(code) - skip
98
                if op is MAX_REPEAT:
99
                    emit(OPCODES[MAX_UNTIL])
100
                else:
101
                    emit(OPCODES[MIN_UNTIL])
102
        elif op is SUBPATTERN:
103
            if av[0]:
104
                emit(OPCODES[MARK])
105
                emit((av[0]-1)*2)
106
            # _compile_info(code, av[1], flags)
107
            _compile(code, av[1], flags)
108
            if av[0]:
109
                emit(OPCODES[MARK])
110
                emit((av[0]-1)*2+1)
111
        elif op in SUCCESS_CODES:
112
            emit(OPCODES[op])
113
        elif op in ASSERT_CODES:
114
            emit(OPCODES[op])
115
            skip = _len(code); emit(0)
116
            if av[0] >= 0:
117
                emit(0) # look ahead
118
            else:
119
                lo, hi = av[1].getwidth()
120
                if lo != hi:
121
                    raise error, "look-behind requires fixed-width pattern"
122
                emit(lo) # look behind
123
            _compile(code, av[1], flags)
124
            emit(OPCODES[SUCCESS])
125
            code[skip] = _len(code) - skip
126
        elif op is CALL:
127
            emit(OPCODES[op])
128
            skip = _len(code); emit(0)
129
            _compile(code, av, flags)
130
            emit(OPCODES[SUCCESS])
131
            code[skip] = _len(code) - skip
132
        elif op is AT:
133
            emit(OPCODES[op])
134
            if flags & SRE_FLAG_MULTILINE:
135
                av = AT_MULTILINE.get(av, av)
136
            if flags & SRE_FLAG_LOCALE:
137
                av = AT_LOCALE.get(av, av)
138
            elif flags & SRE_FLAG_UNICODE:
139
                av = AT_UNICODE.get(av, av)
140
            emit(ATCODES[av])
141
        elif op is BRANCH:
142
            emit(OPCODES[op])
143
            tail = []
144
            tailappend = tail.append
145
            for av in av[1]:
146
                skip = _len(code); emit(0)
147
                # _compile_info(code, av, flags)
148
                _compile(code, av, flags)
149
                emit(OPCODES[JUMP])
150
                tailappend(_len(code)); emit(0)
151
                code[skip] = _len(code) - skip
152
            emit(0) # end of branch
153
            for tail in tail:
154
                code[tail] = _len(code) - tail
155
        elif op is CATEGORY:
156
            emit(OPCODES[op])
157
            if flags & SRE_FLAG_LOCALE:
158
                av = CH_LOCALE[av]
159
            elif flags & SRE_FLAG_UNICODE:
160
                av = CH_UNICODE[av]
161
            emit(CHCODES[av])
162
        elif op is GROUPREF:
163
            if flags & SRE_FLAG_IGNORECASE:
164
                emit(OPCODES[OP_IGNORE[op]])
165
            else:
166
                emit(OPCODES[op])
167
            emit(av-1)
168
        elif op is GROUPREF_EXISTS:
169
            emit(OPCODES[op])
170
            emit(av[0]-1)
171
            skipyes = _len(code); emit(0)
172
            _compile(code, av[1], flags)
173
            if av[2]:
174
                emit(OPCODES[JUMP])
175
                skipno = _len(code); emit(0)
176
                code[skipyes] = _len(code) - skipyes + 1
177
                _compile(code, av[2], flags)
178
                code[skipno] = _len(code) - skipno
179
            else:
180
                code[skipyes] = _len(code) - skipyes + 1
181
        else:
182
            raise ValueError, ("unsupported operand type", op)
183
184
def _compile_charset(charset, flags, code, fixup=None):
185
    # compile charset subprogram
186
    emit = code.append
187
    if fixup is None:
188
        fixup = _identityfunction
189
    for op, av in _optimize_charset(charset, fixup):
190
        emit(OPCODES[op])
191
        if op is NEGATE:
192
            pass
193
        elif op is LITERAL:
194
            emit(fixup(av))
195
        elif op is RANGE:
196
            emit(fixup(av[0]))
197
            emit(fixup(av[1]))
198
        elif op is CHARSET:
199
            code.extend(av)
200
        elif op is BIGCHARSET:
201
            code.extend(av)
202
        elif op is CATEGORY:
203
            if flags & SRE_FLAG_LOCALE:
204
                emit(CHCODES[CH_LOCALE[av]])
205
            elif flags & SRE_FLAG_UNICODE:
206
                emit(CHCODES[CH_UNICODE[av]])
207
            else:
208
                emit(CHCODES[av])
209
        else:
210
            raise error, "internal: unsupported set operator"
211
    emit(OPCODES[FAILURE])
212
213
def _optimize_charset(charset, fixup):
214
    # internal: optimize character set
215
    out = []
216
    outappend = out.append
217
    charmap = [0]*256
218
    try:
219
        for op, av in charset:
220
            if op is NEGATE:
221
                outappend((op, av))
222
            elif op is LITERAL:
223
                charmap[fixup(av)] = 1
224
            elif op is RANGE:
225
                for i in range(fixup(av[0]), fixup(av[1])+1):
226
                    charmap[i] = 1
227
            elif op is CATEGORY:
228
                # XXX: could append to charmap tail
229
                return charset # cannot compress
230
    except IndexError:
231
        # character set contains unicode characters
232
        return _optimize_unicode(charset, fixup)
233
    # compress character map
234
    i = p = n = 0
235
    runs = []
236
    runsappend = runs.append
237
    for c in charmap:
238
        if c:
239
            if n == 0:
240
                p = i
241
            n = n + 1
242
        elif n:
243
            runsappend((p, n))
244
            n = 0
245
        i = i + 1
246
    if n:
247
        runsappend((p, n))
248
    if len(runs) <= 2:
249
        # use literal/range
250
        for p, n in runs:
251
            if n == 1:
252
                outappend((LITERAL, p))
253
            else:
254
                outappend((RANGE, (p, p+n-1)))
255
        if len(out) < len(charset):
256
            return out
257
    else:
258
        # use bitmap
259
        data = _mk_bitmap(charmap)
260
        outappend((CHARSET, data))
261
        return out
262
    return charset
263
264
def _mk_bitmap(bits):
265
    data = []
266
    dataappend = data.append
267
    if _sre.CODESIZE == 2:
268
        start = (1, 0)
269
    else:
270
        start = (1L, 0L)
271
    m, v = start
272
    for c in bits:
273
        if c:
274
            v = v + m
275
        m = m + m
276
        if m > MAXCODE:
277
            dataappend(v)
278
            m, v = start
279
    return data
280
281
# To represent a big charset, first a bitmap of all characters in the
282
# set is constructed. Then, this bitmap is sliced into chunks of 256
283
# characters, duplicate chunks are eliminated, and each chunk is
284
# given a number. In the compiled expression, the charset is
285
# represented by a 16-bit word sequence, consisting of one word for
286
# the number of different chunks, a sequence of 256 bytes (128 words)
287
# of chunk numbers indexed by their original chunk position, and a
288
# sequence of chunks (16 words each).
289
290
# Compression is normally good: in a typical charset, large ranges of
291
# Unicode will be either completely excluded (e.g. if only cyrillic
292
# letters are to be matched), or completely included (e.g. if large
293
# subranges of Kanji match). These ranges will be represented by
294
# chunks of all one-bits or all zero-bits.
295
296
# Matching can be also done efficiently: the more significant byte of
297
# the Unicode character is an index into the chunk number, and the
298
# less significant byte is a bit index in the chunk (just like the
299
# CHARSET matching).
300
301
# In UCS-4 mode, the BIGCHARSET opcode still supports only subsets
302
# of the basic multilingual plane; an efficient representation
303
# for all of UTF-16 has not yet been developed. This means,
304
# in particular, that negated charsets cannot be represented as
305
# bigcharsets.
306
307
def _optimize_unicode(charset, fixup):
308
    try:
309
        import array
310
    except ImportError:
311
        return charset
312
    charmap = [0]*65536
313
    negate = 0
314
    try:
315
        for op, av in charset:
316
            if op is NEGATE:
317
                negate = 1
318
            elif op is LITERAL:
319
                charmap[fixup(av)] = 1
320
            elif op is RANGE:
321
                for i in xrange(fixup(av[0]), fixup(av[1])+1):
322
                    charmap[i] = 1
323
            elif op is CATEGORY:
324
                # XXX: could expand category
325
                return charset # cannot compress
326
    except IndexError:
327
        # non-BMP characters
328
        return charset
329
    if negate:
330
        if sys.maxunicode != 65535:
331
            # XXX: negation does not work with big charsets
332
            return charset
333
        for i in xrange(65536):
334
            charmap[i] = not charmap[i]
335
    comps = {}
336
    mapping = [0]*256
337
    block = 0
338
    data = []
339
    for i in xrange(256):
340
        chunk = tuple(charmap[i*256:(i+1)*256])
341
        new = comps.setdefault(chunk, block)
342
        mapping[i] = new
343
        if new == block:
344
            block = block + 1
345
            data = data + _mk_bitmap(chunk)
346
    header = [block]
347
    if _sre.CODESIZE == 2:
348
        code = 'H'
349
    else:
350
        code = 'I'
351
    # Convert block indices to byte array of 256 bytes
352
    mapping = array.array('b', mapping).tostring()
353
    # Convert byte array to word array
354
    mapping = array.array(code, mapping)
355
    assert mapping.itemsize == _sre.CODESIZE
356
    header = header + mapping.tolist()
357
    data[0:0] = header
358
    return [(BIGCHARSET, data)]
359
360
def _simple(av):
361
    # check if av is a "simple" operator
362
    lo, hi = av[2].getwidth()
363
    if lo == 0 and hi == MAXREPEAT:
364
        raise error, "nothing to repeat"
365
    return lo == hi == 1 and av[2][0][0] != SUBPATTERN
366
9e-06 sec 367
def _compile_info(code, pattern, flags):
368
    # internal: compile an info block.  in the current version,
369
    # this contains min/max pattern width, and an optional literal
370
    # prefix or a character map
2e-06 sec 371
    lo, hi = pattern.getwidth()
3e-06 sec 372
    if lo == 0:
373
        return # not worth it
374
    # look for a literal prefix
2e-06 sec 375
    prefix = []
1e-06 sec 376
    prefixappend = prefix.append
3e-06 sec 377
    prefix_skip = 0
1e-06 sec 378
    charset = [] # not used
2e-06 sec 379
    charsetappend = charset.append
3e-06 sec 380
    if not (flags & SRE_FLAG_IGNORECASE):
381
        # look for literal prefix
382
        for op, av in pattern.data:
383
            if op is LITERAL:
384
                if len(prefix) == prefix_skip:
385
                    prefix_skip = prefix_skip + 1
386
                prefixappend(av)
387
            elif op is SUBPATTERN and len(av[1]) == 1:
388
                op, av = av[1][0]
389
                if op is LITERAL:
390
                    prefixappend(av)
391
                else:
392
                    break
393
            else:
394
                break
395
        # if no prefix, look for charset prefix
396
        if not prefix and pattern.data:
397
            op, av = pattern.data[0]
398
            if op is SUBPATTERN and av[1]:
399
                op, av = av[1][0]
400
                if op is LITERAL:
401
                    charsetappend((op, av))
402
                elif op is BRANCH:
403
                    c = []
404
                    cappend = c.append
405
                    for p in av[1]:
406
                        if not p:
407
                            break
408
                        op, av = p[0]
409
                        if op is LITERAL:
410
                            cappend((op, av))
411
                        else:
412
                            break
413
                    else:
414
                        charset = c
415
            elif op is BRANCH:
416
                c = []
417
                cappend = c.append
418
                for p in av[1]:
419
                    if not p:
420
                        break
421
                    op, av = p[0]
422
                    if op is LITERAL:
423
                        cappend((op, av))
424
                    else:
425
                        break
426
                else:
427
                    charset = c
428
            elif op is IN:
429
                charset = av
430
##     if prefix:
431
##         print "*** PREFIX", prefix, prefix_skip
432
##     if charset:
433
##         print "*** CHARSET", charset
434
    # add an info block
3e-06 sec 435
    emit = code.append
3e-06 sec 436
    emit(OPCODES[INFO])
5e-06 sec 437
    skip = len(code); emit(0)
438
    # literal flag
3e-06 sec 439
    mask = 0
2e-06 sec 440
    if prefix:
441
        mask = SRE_INFO_PREFIX
442
        if len(prefix) == prefix_skip == len(pattern.data):
443
            mask = mask + SRE_INFO_LITERAL
3e-06 sec 444
    elif charset:
445
        mask = mask + SRE_INFO_CHARSET
3e-06 sec 446
    emit(mask)
447
    # pattern length
3e-06 sec 448
    if lo < MAXCODE:
4e-06 sec 449
        emit(lo)
450
    else:
451
        emit(MAXCODE)
452
        prefix = prefix[:MAXCODE]
2e-06 sec 453
    if hi < MAXCODE:
3e-06 sec 454
        emit(hi)
455
    else:
456
        emit(0)
457
    # add literal prefix
4e-06 sec 458
    if prefix:
459
        emit(len(prefix)) # length
460
        emit(prefix_skip) # skip
461
        code.extend(prefix)
462
        # generate overlap table
463
        table = [-1] + ([0]*len(prefix))
464
        for i in xrange(len(prefix)):
465
            table[i+1] = table[i]+1
466
            while table[i+1] > 0 and prefix[i] != prefix[table[i+1]-1]:
467
                table[i+1] = table[table[i+1]-1]+1
468
        code.extend(table[1:]) # don't store first entry
4e-06 sec 469
    elif charset:
470
        _compile_charset(charset, flags, code)
4e-06 sec 471
    code[skip] = len(code) - skip
472
473
try:
474
    unicode
475
except NameError:
476
    STRING_TYPES = (type(""),)
477
else:
478
    STRING_TYPES = (type(""), type(unicode("")))
479
2e-05 sec 480
def isstring(obj):
5e-06 sec 481
    for tp in STRING_TYPES:
6e-06 sec 482
        if isinstance(obj, tp):
5e-06 sec 483
            return 1
484
    return 0
485
7e-06 sec 486
def _code(p, flags):
487
2e-06 sec 488
    flags = p.pattern.flags | flags
2e-06 sec 489
    code = []
490
491
    # compile info block
2e-06 sec 492
    _compile_info(code, p, flags)
493
494
    # compile the pattern
3e-06 sec 495
    _compile(code, p.data, flags)
496
2e-06 sec 497
    code.append(OPCODES[SUCCESS])
498
3e-06 sec 499
    return code
500
1.1e-05 sec 501
def compile(p, flags=0):
502
    # internal: convert pattern list to internal format
503
2e-06 sec 504
    if isstring(p):
3e-06 sec 505
        pattern = p
1e-06 sec 506
        p = sre_parse.parse(p, flags)
507
    else:
508
        pattern = None
509
3e-06 sec 510
    code = _code(p, flags)
511
512
    # print code
513
514
    # XXX: <fl> get rid of this limitation!
2e-06 sec 515
    if p.pattern.groups > 100:
516
        raise AssertionError(
517
            "sorry, but this version only supports 100 named groups"
518
            )
519
520
    # map in either direction
3e-06 sec 521
    groupindex = p.pattern.groupdict
2e-06 sec 522
    indexgroup = [None] * p.pattern.groups
5e-06 sec 523
    for k, i in groupindex.items():
524
        indexgroup[i] = k
525
4e-06 sec 526
    return _sre.compile(
3e-06 sec 527
        pattern, flags | p.pattern.flags, code,
3e-06 sec 528
        p.pattern.groups-1,
2e-06 sec 529
        groupindex, indexgroup
530
        )
/usr/lib/python2.6/sre_parse.py
     time      num code
1
#
2
# Secret Labs' Regular Expression Engine
3
#
4
# convert re-style regular expression to sre pattern
5
#
6
# Copyright (c) 1998-2001 by Secret Labs AB.  All rights reserved.
7
#
8
# See the sre.py file for information on usage and redistribution.
9
#
10
11
"""Internal support module for sre"""
12
13
# XXX: show string offset and offending character for all errors
14
15
import sys
16
17
from sre_constants import *
18
19
def set(seq):
20
    s = {}
21
    for elem in seq:
22
        s[elem] = 1
23
    return s
24
25
SPECIAL_CHARS = ".\\[{()*+?^$|"
26
REPEAT_CHARS = "*+?{"
27
28
DIGITS = set("0123456789")
29
30
OCTDIGITS = set("01234567")
31
HEXDIGITS = set("0123456789abcdefABCDEF")
32
33
WHITESPACE = set(" \t\n\r\v\f")
34
35
ESCAPES = {
36
    r"\a": (LITERAL, ord("\a")),
37
    r"\b": (LITERAL, ord("\b")),
38
    r"\f": (LITERAL, ord("\f")),
39
    r"\n": (LITERAL, ord("\n")),
40
    r"\r": (LITERAL, ord("\r")),
41
    r"\t": (LITERAL, ord("\t")),
42
    r"\v": (LITERAL, ord("\v")),
43
    r"\\": (LITERAL, ord("\\"))
44
}
45
46
CATEGORIES = {
47
    r"\A": (AT, AT_BEGINNING_STRING), # start of string
48
    r"\b": (AT, AT_BOUNDARY),
49
    r"\B": (AT, AT_NON_BOUNDARY),
50
    r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]),
51
    r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]),
52
    r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]),
53
    r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]),
54
    r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]),
55
    r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]),
56
    r"\Z": (AT, AT_END_STRING), # end of string
57
}
58
59
FLAGS = {
60
    # standard flags
61
    "i": SRE_FLAG_IGNORECASE,
62
    "L": SRE_FLAG_LOCALE,
63
    "m": SRE_FLAG_MULTILINE,
64
    "s": SRE_FLAG_DOTALL,
65
    "x": SRE_FLAG_VERBOSE,
66
    # extensions
67
    "t": SRE_FLAG_TEMPLATE,
68
    "u": SRE_FLAG_UNICODE,
69
}
70
71
class Pattern:
72
    # master pattern object.  keeps track of global attributes
9e-06 sec 73
    def __init__(self):
3e-06 sec 74
        self.flags = 0
2e-06 sec 75
        self.open = []
3e-06 sec 76
        self.groups = 1
2e-06 sec 77
        self.groupdict = {}
78
    def opengroup(self, name=None):
79
        gid = self.groups
80
        self.groups = gid + 1
81
        if name is not None:
82
            ogid = self.groupdict.get(name, None)
83
            if ogid is not None:
84
                raise error, ("redefinition of group name %s as group %d; "
85
                              "was group %d" % (repr(name), gid,  ogid))
86
            self.groupdict[name] = gid
87
        self.open.append(gid)
88
        return gid
89
    def closegroup(self, gid):
90
        self.open.remove(gid)
91
    def checkgroup(self, gid):
92
        return gid < self.groups and gid not in self.open
93
94
class SubPattern:
95
    # a subpattern, in intermediate form
1.1e-05 sec 96
    def __init__(self, pattern, data=None):
1e-06 sec 97
        self.pattern = pattern
3e-06 sec 98
        if data is None:
2e-06 sec 99
            data = []
1e-06 sec 100
        self.data = data
2e-06 sec 101
        self.width = None
102
    def dump(self, level=0):
103
        nl = 1
104
        seqtypes = type(()), type([])
105
        for op, av in self.data:
106
            print level*"  " + op,; nl = 0
107
            if op == "in":
108
                # member sublanguage
109
                print; nl = 1
110
                for op, a in av:
111
                    print (level+1)*"  " + op, a
112
            elif op == "branch":
113
                print; nl = 1
114
                i = 0
115
                for a in av[1]:
116
                    if i > 0:
117
                        print level*"  " + "or"
118
                    a.dump(level+1); nl = 1
119
                    i = i + 1
120
            elif type(av) in seqtypes:
121
                for a in av:
122
                    if isinstance(a, SubPattern):
123
                        if not nl: print
124
                        a.dump(level+1); nl = 1
125
                    else:
126
                        print a, ; nl = 0
127
            else:
128
                print av, ; nl = 0
129
            if not nl: print
130
    def __repr__(self):
131
        return repr(self.data)
132
    def __len__(self):
133
        return len(self.data)
134
    def __delitem__(self, index):
135
        del self.data[index]
136
    def __getitem__(self, index):
137
        if isinstance(index, slice):
138
            return SubPattern(self.pattern, self.data[index])
139
        return self.data[index]
140
    def __setitem__(self, index, code):
141
        self.data[index] = code
142
    def insert(self, index, code):
143
        self.data.insert(index, code)
2.3e-05 sec 144
    def append(self, code):
5e-06 sec 145
        self.data.append(code)
6e-06 sec 146
    def getwidth(self):
147
        # determine the width (min, max) for this subpattern
2e-06 sec 148
        if self.width:
149
            return self.width
3e-06 sec 150
        lo = hi = 0L
2e-06 sec 151
        UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY)
5e-06 sec 152
        REPEATCODES = (MIN_REPEAT, MAX_REPEAT)
1.1e-05 sec 153
        for op, av in self.data:
8e-06 sec 154
            if op is BRANCH:
155
                i = sys.maxint
156
                j = 0
157
                for av in av[1]:
158
                    l, h = av.getwidth()
159
                    i = min(i, l)
160
                    j = max(j, h)
161
                lo = lo + i
162
                hi = hi + j
7e-06 sec 163
            elif op is CALL:
164
                i, j = av.getwidth()
165
                lo = lo + i
166
                hi = hi + j
7e-06 sec 167
            elif op is SUBPATTERN:
168
                i, j = av[1].getwidth()
169
                lo = lo + i
170
                hi = hi + j
7e-06 sec 171
            elif op in REPEATCODES:
172
                i, j = av[2].getwidth()
173
                lo = lo + long(i) * av[0]
174
                hi = hi + long(j) * av[1]
9e-06 sec 175
            elif op in UNITCODES:
7e-06 sec 176
                lo = lo + 1
1.2e-05 sec 177
                hi = hi + 1
178
            elif op == SUCCESS:
179
                break
3e-06 sec 180
        self.width = int(min(lo, sys.maxint)), int(min(hi, sys.maxint))
1.7e-05 sec 181
        return self.width
182
183
class Tokenizer:
1.2e-05 sec 184
    def __init__(self, string):
2e-06 sec 185
        self.string = string
3e-06 sec 186
        self.index = 0
2e-06 sec 187
        self.__next()
5.2e-05 sec 188
    def __next(self):
1.6e-05 sec 189
        if self.index >= len(self.string):
8e-06 sec 190
            self.next = None
6e-06 sec 191
            return
2.2e-05 sec 192
        char = self.string[self.index]
1.9e-05 sec 193
        if char[0] == "\\":
194
            try:
195
                c = self.string[self.index + 1]
196
            except IndexError:
197
                raise error, "bogus escape (end of line)"
198
            char = char + c
1.7e-05 sec 199
        self.index = self.index + len(char)
2e-05 sec 200
        self.next = char
3e-05 sec 201
    def match(self, char, skip=1):
1e-05 sec 202
        if char == self.next:
2e-06 sec 203
            if skip:
2e-06 sec 204
                self.__next()
2e-06 sec 205
            return 1
1.2e-05 sec 206
        return 0
3.6e-05 sec 207
    def get(self):
1.1e-05 sec 208
        this = self.next
1.6e-05 sec 209
        self.__next()
1.6e-05 sec 210
        return this
211
    def tell(self):
212
        return self.index, self.next
213
    def seek(self, index):
214
        self.index, self.next = index
215
216
def isident(char):
217
    return "a" <= char <= "z" or "A" <= char <= "Z" or char == "_"
218
219
def isdigit(char):
220
    return "0" <= char <= "9"
221
222
def isname(name):
223
    # check that group name is a valid string
224
    if not isident(name[0]):
225
        return False
226
    for char in name[1:]:
227
        if not isident(char) and not isdigit(char):
228
            return False
229
    return True
230
231
def _class_escape(source, escape):
232
    # handle escape code inside character class
233
    code = ESCAPES.get(escape)
234
    if code:
235
        return code
236
    code = CATEGORIES.get(escape)
237
    if code:
238
        return code
239
    try:
240
        c = escape[1:2]
241
        if c == "x":
242
            # hexadecimal escape (exactly two digits)
243
            while source.next in HEXDIGITS and len(escape) < 4:
244
                escape = escape + source.get()
245
            escape = escape[2:]
246
            if len(escape) != 2:
247
                raise error, "bogus escape: %s" % repr("\\" + escape)
248
            return LITERAL, int(escape, 16) & 0xff
249
        elif c in OCTDIGITS:
250
            # octal escape (up to three digits)
251
            while source.next in OCTDIGITS and len(escape) < 4:
252
                escape = escape + source.get()
253
            escape = escape[1:]
254
            return LITERAL, int(escape, 8) & 0xff
255
        elif c in DIGITS:
256
            raise error, "bogus escape: %s" % repr(escape)
257
        if len(escape) == 2:
258
            return LITERAL, ord(escape[1])
259
    except ValueError:
260
        pass
261
    raise error, "bogus escape: %s" % repr(escape)
262
263
def _escape(source, escape, state):
264
    # handle escape code in expression
265
    code = CATEGORIES.get(escape)
266
    if code:
267
        return code
268
    code = ESCAPES.get(escape)
269
    if code:
270
        return code
271
    try:
272
        c = escape[1:2]
273
        if c == "x":
274
            # hexadecimal escape
275
            while source.next in HEXDIGITS and len(escape) < 4:
276
                escape = escape + source.get()
277
            if len(escape) != 4:
278
                raise ValueError
279
            return LITERAL, int(escape[2:], 16) & 0xff
280
        elif c == "0":
281
            # octal escape
282
            while source.next in OCTDIGITS and len(escape) < 4:
283
                escape = escape + source.get()
284
            return LITERAL, int(escape[1:], 8) & 0xff
285
        elif c in DIGITS:
286
            # octal escape *or* decimal group reference (sigh)
287
            if source.next in DIGITS:
288
                escape = escape + source.get()
289
                if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and
290
                    source.next in OCTDIGITS):
291
                    # got three octal digits; this is an octal escape
292
                    escape = escape + source.get()
293
                    return LITERAL, int(escape[1:], 8) & 0xff
294
            # not an octal escape, so this is a group reference
295
            group = int(escape[1:])
296
            if group < state.groups:
297
                if not state.checkgroup(group):
298
                    raise error, "cannot refer to open group"
299
                return GROUPREF, group
300
            raise ValueError
301
        if len(escape) == 2:
302
            return LITERAL, ord(escape[1])
303
    except ValueError:
304
        pass
305
    raise error, "bogus escape: %s" % repr(escape)
306
7e-06 sec 307
def _parse_sub(source, state, nested=1):
308
    # parse an alternation: a|b|c
309
2e-06 sec 310
    items = []
2e-06 sec 311
    itemsappend = items.append
3e-06 sec 312
    sourcematch = source.match
3e-06 sec 313
    while 1:
2e-06 sec 314
        itemsappend(_parse(source, state))
3e-06 sec 315
        if sourcematch("|"):
316
            continue
2e-06 sec 317
        if not nested:
2e-06 sec 318
            break
319
        if not source.next or sourcematch(")", 0):
320
            break
321
        else:
322
            raise error, "pattern not properly closed"
323
2e-06 sec 324
    if len(items) == 1:
3e-06 sec 325
        return items[0]
326
327
    subpattern = SubPattern(state)
328
    subpatternappend = subpattern.append
329
330
    # check if all items share a common prefix
331
    while 1:
332
        prefix = None
333
        for item in items:
334
            if not item:
335
                break
336
            if prefix is None:
337
                prefix = item[0]
338
            elif item[0] != prefix:
339
                break
340
        else:
341
            # all subitems start with a common "prefix".
342
            # move it out of the branch
343
            for item in items:
344
                del item[0]
345
            subpatternappend(prefix)
346
            continue # check next one
347
        break
348
349
    # check if the branch can be replaced by a character set
350
    for item in items:
351
        if len(item) != 1 or item[0][0] != LITERAL:
352
            break
353
    else:
354
        # we can store this as a character set instead of a
355
        # branch (the compiler may optimize this even more)
356
        set = []
357
        setappend = set.append
358
        for item in items:
359
            setappend(item[0])
360
        subpatternappend((IN, set))
361
        return subpattern
362
363
    subpattern.append((BRANCH, (None, items)))
364
    return subpattern
365
366
def _parse_sub_cond(source, state, condgroup):
367
    item_yes = _parse(source, state)
368
    if source.match("|"):
369
        item_no = _parse(source, state)
370
        if source.match("|"):
371
            raise error, "conditional backref with more than two branches"
372
    else:
373
        item_no = None
374
    if source.next and not source.match(")", 0):
375
        raise error, "pattern not properly closed"
376
    subpattern = SubPattern(state)
377
    subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no)))
378
    return subpattern
379
380
_PATTERNENDERS = set("|)")
381
_ASSERTCHARS = set("=!<")
382
_LOOKBEHINDASSERTCHARS = set("=!")
383
_REPEATCODES = set([MIN_REPEAT, MAX_REPEAT])
384
7e-06 sec 385
def _parse(source, state):
386
    # parse a simple pattern
2e-06 sec 387
    subpattern = SubPattern(state)
388
389
    # precompute constants into local variables
2e-06 sec 390
    subpatternappend = subpattern.append
3e-06 sec 391
    sourceget = source.get
3e-06 sec 392
    sourcematch = source.match
3e-06 sec 393
    _len = len
2e-06 sec 394
    PATTERNENDERS = _PATTERNENDERS
2e-06 sec 395
    ASSERTCHARS = _ASSERTCHARS
2e-06 sec 396
    LOOKBEHINDASSERTCHARS = _LOOKBEHINDASSERTCHARS
2e-06 sec 397
    REPEATCODES = _REPEATCODES
398
2e-06 sec 399
    while 1:
400
1.3e-05 sec 401
        if source.next in PATTERNENDERS:
402
            break # end of subpattern
1.2e-05 sec 403
        this = sourceget()
9e-06 sec 404
        if this is None:
2e-06 sec 405
            break # end of pattern
406
9e-06 sec 407
        if state.flags & SRE_FLAG_VERBOSE:
408
            # skip whitespace and comments
409
            if this in WHITESPACE:
410
                continue
411
            if this == "#":
412
                while 1:
413
                    this = sourceget()
414
                    if this in (None, "\n"):
415
                        break
416
                continue
417
1.4e-05 sec 418
        if this and this[0] not in SPECIAL_CHARS:
8e-06 sec 419
            subpatternappend((LITERAL, ord(this)))
420
5e-06 sec 421
        elif this == "[":
422
            # character set
423
            set = []
424
            setappend = set.append
425
##          if sourcematch(":"):
426
##              pass # handle character classes
427
            if sourcematch("^"):
428
                setappend((NEGATE, None))
429
            # check remaining characters
430
            start = set[:]
431
            while 1:
432
                this = sourceget()
433
                if this == "]" and set != start:
434
                    break
435
                elif this and this[0] == "\\":
436
                    code1 = _class_escape(source, this)
437
                elif this:
438
                    code1 = LITERAL, ord(this)
439
                else:
440
                    raise error, "unexpected end of regular expression"
441
                if sourcematch("-"):
442
                    # potential range
443
                    this = sourceget()
444
                    if this == "]":
445
                        if code1[0] is IN:
446
                            code1 = code1[1][0]
447
                        setappend(code1)
448
                        setappend((LITERAL, ord("-")))
449
                        break
450
                    elif this:
451
                        if this[0] == "\\":
452
                            code2 = _class_escape(source, this)
453
                        else:
454
                            code2 = LITERAL, ord(this)
455
                        if code1[0] != LITERAL or code2[0] != LITERAL:
456
                            raise error, "bad character range"
457
                        lo = code1[1]
458
                        hi = code2[1]
459
                        if hi < lo:
460
                            raise error, "bad character range"
461
                        setappend((RANGE, (lo, hi)))
462
                    else:
463
                        raise error, "unexpected end of regular expression"
464
                else:
465
                    if code1[0] is IN:
466
                        code1 = code1[1][0]
467
                    setappend(code1)
468
469
            # XXX: <fl> should move set optimization to compiler!
470
            if _len(set)==1 and set[0][0] is LITERAL:
471
                subpatternappend(set[0]) # optimization
472
            elif _len(set)==2 and set[0][0] is NEGATE and set[1][0] is LITERAL:
473
                subpatternappend((NOT_LITERAL, set[1][1])) # optimization
474
            else:
475
                # XXX: <fl> should add charmap optimization here
476
                subpatternappend((IN, set))
477
3e-06 sec 478
        elif this and this[0] in REPEAT_CHARS:
479
            # repeat previous item
480
            if this == "?":
481
                min, max = 0, 1
482
            elif this == "*":
483
                min, max = 0, MAXREPEAT
484
485
            elif this == "+":
486
                min, max = 1, MAXREPEAT
487
            elif this == "{":
488
                if source.next == "}":
489
                    subpatternappend((LITERAL, ord(this)))
490
                    continue
491
                here = source.tell()
492
                min, max = 0, MAXREPEAT
493
                lo = hi = ""
494
                while source.next in DIGITS:
495
                    lo = lo + source.get()
496
                if sourcematch(","):
497
                    while source.next in DIGITS:
498
                        hi = hi + sourceget()
499
                else:
500
                    hi = lo
501
                if not sourcematch("}"):
502
                    subpatternappend((LITERAL, ord(this)))
503
                    source.seek(here)
504
                    continue
505
                if lo:
506
                    min = int(lo)
507
                if hi:
508
                    max = int(hi)
509
                if max < min:
510
                    raise error, "bad repeat interval"
511
            else:
512
                raise error, "not supported"
513
            # figure out which item to repeat
514
            if subpattern:
515
                item = subpattern[-1:]
516
            else:
517
                item = None
518
            if not item or (_len(item) == 1 and item[0][0] == AT):
519
                raise error, "nothing to repeat"
520
            if item[0][0] in REPEATCODES:
521
                raise error, "multiple repeat"
522
            if sourcematch("?"):
523
                subpattern[-1] = (MIN_REPEAT, (min, max, item))
524
            else:
525
                subpattern[-1] = (MAX_REPEAT, (min, max, item))
526
6e-06 sec 527
        elif this == ".":
528
            subpatternappend((ANY, None))
529
4e-06 sec 530
        elif this == "(":
3e-06 sec 531
            group = 1
3e-06 sec 532
            name = None
3e-06 sec 533
            condgroup = None
2e-06 sec 534
            if sourcematch("?"):
3e-06 sec 535
                group = 0
536
                # options
2e-06 sec 537
                if sourcematch("P"):
538
                    # python extensions
539
                    if sourcematch("<"):
540
                        # named group: skip forward to end of name
541
                        name = ""
542
                        while 1:
543
                            char = sourceget()
544
                            if char is None:
545
                                raise error, "unterminated name"
546
                            if char == ">":
547
                                break
548
                            name = name + char
549
                        group = 1
550
                        if not isname(name):
551
                            raise error, "bad character in group name"
552
                    elif sourcematch("="):
553
                        # named backreference
554
                        name = ""
555
                        while 1:
556
                            char = sourceget()
557
                            if char is None:
558
                                raise error, "unterminated name"
559
                            if char == ")":
560
                                break
561
                            name = name + char
562
                        if not isname(name):
563
                            raise error, "bad character in group name"
564
                        gid = state.groupdict.get(name)
565
                        if gid is None:
566
                            raise error, "unknown group name"
567
                        subpatternappend((GROUPREF, gid))
568
                        continue
569
                    else:
570
                        char = sourceget()
571
                        if char is None:
572
                            raise error, "unexpected end of pattern"
573
                        raise error, "unknown specifier: ?P%s" % char
5e-06 sec 574
                elif sourcematch(":"):
575
                    # non-capturing group
576
                    group = 2
4e-06 sec 577
                elif sourcematch("#"):
578
                    # comment
579
                    while 1:
580
                        if source.next is None or source.next == ")":
581
                            break
582
                        sourceget()
583
                    if not sourcematch(")"):
584
                        raise error, "unbalanced parenthesis"
585
                    continue
5e-06 sec 586
                elif source.next in ASSERTCHARS:
587
                    # lookahead assertions
588
                    char = sourceget()
589
                    dir = 1
590
                    if char == "<":
591
                        if source.next not in LOOKBEHINDASSERTCHARS:
592
                            raise error, "syntax error"
593
                        dir = -1 # lookbehind
594
                        char = sourceget()
595
                    p = _parse_sub(source, state)
596
                    if not sourcematch(")"):
597
                        raise error, "unbalanced parenthesis"
598
                    if char == "=":
599
                        subpatternappend((ASSERT, (dir, p)))
600
                    else:
601
                        subpatternappend((ASSERT_NOT, (dir, p)))
602
                    continue
4e-06 sec 603
                elif sourcematch("("):
604
                    # conditional backreference group
605
                    condname = ""
606
                    while 1:
607
                        char = sourceget()
608
                        if char is None:
609
                            raise error, "unterminated name"
610
                        if char == ")":
611
                            break
612
                        condname = condname + char
613
                    group = 2
614
                    if isname(condname):
615
                        condgroup = state.groupdict.get(condname)
616
                        if condgroup is None:
617
                            raise error, "unknown group name"
618
                    else:
619
                        try:
620
                            condgroup = int(condname)
621
                        except ValueError:
622
                            raise error, "bad character in group name"
623
                else:
624
                    # flags
5e-06 sec 625
                    if not source.next in FLAGS:
626
                        raise error, "unexpected end of pattern"
1.1e-05 sec 627
                    while source.next in FLAGS:
4e-06 sec 628
                        state.flags = state.flags | FLAGS[sourceget()]
6e-06 sec 629
            if group:
630
                # parse group contents
631
                if group == 2:
632
                    # anonymous group
633
                    group = None
634
                else:
635
                    group = state.opengroup(name)
636
                if condgroup:
637
                    p = _parse_sub_cond(source, state, condgroup)
638
                else:
639
                    p = _parse_sub(source, state)
640
                if not sourcematch(")"):
641
                    raise error, "unbalanced parenthesis"
642
                if group is not None:
643
                    state.closegroup(group)
644
                subpatternappend((SUBPATTERN, (group, p)))
645
            else:
5e-06 sec 646
                while 1:
3e-06 sec 647
                    char = sourceget()
4e-06 sec 648
                    if char is None:
649
                        raise error, "unexpected end of pattern"
5e-06 sec 650
                    if char == ")":
3e-06 sec 651
                        break
652
                    raise error, "unknown extension"
653
654
        elif this == "^":
655
            subpatternappend((AT, AT_BEGINNING))
656
657
        elif this == "$":
658
            subpattern.append((AT, AT_END))
659
660
        elif this and this[0] == "\\":
661
            code = _escape(source, this, state)
662
            subpatternappend(code)
663
664
        else:
665
            raise error, "parser error"
666
3e-06 sec 667
    return subpattern
668
1e-05 sec 669
def parse(str, flags=0, pattern=None):
670
    # parse 're' pattern into list of (opcode, argument) tuples
671
2e-06 sec 672
    source = Tokenizer(str)
673
2e-06 sec 674
    if pattern is None:
2e-06 sec 675
        pattern = Pattern()
2e-06 sec 676
    pattern.flags = flags
2e-06 sec 677
    pattern.str = str
678
2e-06 sec 679
    p = _parse_sub(source, pattern, 0)
680
3e-06 sec 681
    tail = source.get()
2e-06 sec 682
    if tail == ")":
683
        raise error, "unbalanced parenthesis"
3e-06 sec 684
    elif tail:
685
        raise error, "bogus characters at end of regular expression"
686
2e-06 sec 687
    if flags & SRE_FLAG_DEBUG:
688
        p.dump()
689
3e-06 sec 690
    if not (flags & SRE_FLAG_VERBOSE) and p.pattern.flags & SRE_FLAG_VERBOSE:
691
        # the VERBOSE flag was switched on inside the pattern.  to be
692
        # on the safe side, we'll parse the whole thing again...
693
        return parse(str, p.pattern.flags)
694
4e-06 sec 695
    return p
696
697
def parse_template(source, pattern):
698
    # parse 're' replacement string into list of literals and
699
    # group references
700
    s = Tokenizer(source)
701
    sget = s.get
702
    p = []
703
    a = p.append
704
    def literal(literal, p=p, pappend=a):
705
        if p and p[-1][0] is LITERAL:
706
            p[-1] = LITERAL, p[-1][1] + literal
707
        else:
708
            pappend((LITERAL, literal))
709
    sep = source[:0]
710
    if type(sep) is type(""):
711
        makechar = chr
712
    else:
713
        makechar = unichr
714
    while 1:
715
        this = sget()
716
        if this is None:
717
            break # end of replacement string
718
        if this and this[0] == "\\":
719
            # group
720
            c = this[1:2]
721
            if c == "g":
722
                name = ""
723
                if s.match("<"):
724
                    while 1:
725
                        char = sget()
726
                        if char is None:
727
                            raise error, "unterminated group name"
728
                        if char == ">":
729
                            break
730
                        name = name + char
731
                if not name:
732
                    raise error, "bad group name"
733
                try:
734
                    index = int(name)
735
                    if index < 0:
736
                        raise error, "negative group number"
737
                except ValueError:
738
                    if not isname(name):
739
                        raise error, "bad character in group name"
740
                    try:
741
                        index = pattern.groupindex[name]
742
                    except KeyError:
743
                        raise IndexError, "unknown group name"
744
                a((MARK, index))
745
            elif c == "0":
746
                if s.next in OCTDIGITS:
747
                    this = this + sget()
748
                    if s.next in OCTDIGITS:
749
                        this = this + sget()
750
                literal(makechar(int(this[1:], 8) & 0xff))
751
            elif c in DIGITS:
752
                isoctal = False
753
                if s.next in DIGITS:
754
                    this = this + sget()
755
                    if (c in OCTDIGITS and this[2] in OCTDIGITS and
756
                        s.next in OCTDIGITS):
757
                        this = this + sget()
758
                        isoctal = True
759
                        literal(makechar(int(this[1:], 8) & 0xff))
760
                if not isoctal:
761
                    a((MARK, int(this[1:])))
762
            else:
763
                try:
764
                    this = makechar(ESCAPES[this][1])
765
                except KeyError:
766
                    pass
767
                literal(this)
768
        else:
769
            literal(this)
770
    # convert template to groups and literals lists
771
    i = 0
772
    groups = []
773
    groupsappend = groups.append
774
    literals = [None] * len(p)
775
    for c, s in p:
776
        if c is MARK:
777
            groupsappend((i, s))
778
            # literal[i] is already None
779
        else:
780
            literals[i] = s
781
        i = i + 1
782
    return groups, literals
783
784
def expand_template(template, match):
785
    g = match.group
786
    sep = match.string[:0]
787
    groups, literals = template
788
    literals = literals[:]
789
    try:
790
        for index, group in groups:
791
            literals[index] = s = g(group)
792
            if s is None:
793
                raise error, "unmatched group"
794
    except IndexError:
795
        raise error, "invalid group reference"
796
    return sep.join(literals)
/usr/lib/python2.6/stat.py
     time      num code
1
"""Constants/functions for interpreting results of os.stat() and os.lstat().
2
3
Suggested usage: from stat import *
4
"""
5
6
# Indices for stat struct members in the tuple returned by os.stat()
7
8
ST_MODE  = 0
9
ST_INO   = 1
10
ST_DEV   = 2
11
ST_NLINK = 3
12
ST_UID   = 4
13
ST_GID   = 5
14
ST_SIZE  = 6
15
ST_ATIME = 7
16
ST_MTIME = 8
17
ST_CTIME = 9
18
19
# Extract bits from the mode
20
21
def S_IMODE(mode):
22
    return mode & 07777
23
7e-06 sec 24
def S_IFMT(mode):
2e-06 sec 25
    return mode & 0170000
26
27
# Constants used as S_IFMT() for various file types
28
# (not all are implemented on all systems)
29
30
S_IFDIR  = 0040000
31
S_IFCHR  = 0020000
32
S_IFBLK  = 0060000
33
S_IFREG  = 0100000
34
S_IFIFO  = 0010000
35
S_IFLNK  = 0120000
36
S_IFSOCK = 0140000
37
38
# Functions to test for each file type
39
40
def S_ISDIR(mode):
41
    return S_IFMT(mode) == S_IFDIR
42
43
def S_ISCHR(mode):
44
    return S_IFMT(mode) == S_IFCHR
45
46
def S_ISBLK(mode):
47
    return S_IFMT(mode) == S_IFBLK
48
1.4e-05 sec 49
def S_ISREG(mode):
2e-06 sec 50
    return S_IFMT(mode) == S_IFREG
51
52
def S_ISFIFO(mode):
53
    return S_IFMT(mode) == S_IFIFO
54
55
def S_ISLNK(mode):
56
    return S_IFMT(mode) == S_IFLNK
57
58
def S_ISSOCK(mode):
59
    return S_IFMT(mode) == S_IFSOCK
60
61
# Names for permission bits
62
63
S_ISUID = 04000
64
S_ISGID = 02000
65
S_ENFMT = S_ISGID
66
S_ISVTX = 01000
67
S_IREAD = 00400
68
S_IWRITE = 00200
69
S_IEXEC = 00100
70
S_IRWXU = 00700
71
S_IRUSR = 00400
72
S_IWUSR = 00200
73
S_IXUSR = 00100
74
S_IRWXG = 00070
75
S_IRGRP = 00040
76
S_IWGRP = 00020
77
S_IXGRP = 00010
78
S_IRWXO = 00007
79
S_IROTH = 00004
80
S_IWOTH = 00002
81
S_IXOTH = 00001
82
83
# Names for file flags
84
85
UF_NODUMP    = 0x00000001
86
UF_IMMUTABLE = 0x00000002
87
UF_APPEND    = 0x00000004
88
UF_OPAQUE    = 0x00000008
89
UF_NOUNLINK  = 0x00000010
90
SF_ARCHIVED  = 0x00010000
91
SF_IMMUTABLE = 0x00020000
92
SF_APPEND    = 0x00040000
93
SF_NOUNLINK  = 0x00100000
94
SF_SNAPSHOT  = 0x00200000
/usr/lib/python2.6/threading.py
     time      num code
1
"""Thread module emulating a subset of Java's threading model."""
2
3
import sys as _sys
4
5
try:
6
    import thread
7
except ImportError:
8
    del _sys.modules[__name__]
9
    raise
10
11
import warnings
12
13
from functools import wraps
14
from time import time as _time, sleep as _sleep
15
from traceback import format_exc as _format_exc
16
from collections import deque
17
18
# Note regarding PEP 8 compliant aliases
19
#  This threading model was originally inspired by Java, and inherited
20
# the convention of camelCase function and method names from that
21
# language. While those names are not in any imminent danger of being
22
# deprecated, starting with Python 2.6, the module now provides a
23
# PEP 8 compliant alias for any such method name.
24
# Using the new PEP 8 compliant names also facilitates substitution
25
# with the multiprocessing module, which doesn't provide the old
26
# Java inspired names.
27
28
29
# Rename some stuff so "from threading import *" is safe
30
__all__ = ['activeCount', 'active_count', 'Condition', 'currentThread',
31
           'current_thread', 'enumerate', 'Event',
32
           'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread',
33
           'Timer', 'setprofile', 'settrace', 'local', 'stack_size']
34
35
_start_new_thread = thread.start_new_thread
36
_allocate_lock = thread.allocate_lock
37
_get_ident = thread.get_ident
38
ThreadError = thread.error
39
del thread
40
41
42
# sys.exc_clear is used to work around the fact that except blocks
43
# don't fully clear the exception until 3.0.
44
warnings.filterwarnings('ignore', category=DeprecationWarning,
45
                        module='threading', message='sys.exc_clear')
46
47
# Debug support (adapted from ihooks.py).
48
# All the major classes here derive from _Verbose.  We force that to
49
# be a new-style class so that all the major classes here are new-style.
50
# This helps debugging (type(instance) is more revealing for instances
51
# of new-style classes).
52
53
_VERBOSE = False
54
55
if __debug__:
56
57
    class _Verbose(object):
58
0.000297 sec 59
        def __init__(self, verbose=None):
7.8e-05 sec 60
            if verbose is None:
8.1e-05 sec 61
                verbose = _VERBOSE
8.2e-05 sec 62
            self.__verbose = verbose
63
0.000282 sec 64
        def _note(self, format, *args):
6.9e-05 sec 65
            if self.__verbose:
66
                format = format % args
67
                format = "%s: %s\n" % (
68
                    current_thread().name, format)
69
                _sys.stderr.write(format)
70
71
else:
72
    # Disable this when using "python -O"
73
    class _Verbose(object):
74
        def __init__(self, verbose=None):
75
            pass
76
        def _note(self, *args):
77
            pass
78
79
# Support for profile and trace hooks
80
81
_profile_hook = None
82
_trace_hook = None
83
84
def setprofile(func):
85
    global _profile_hook
86
    _profile_hook = func
87
88
def settrace(func):
89
    global _trace_hook
90
    _trace_hook = func
91
92
# Synchronization classes
93
94
Lock = _allocate_lock
95
96
def RLock(*args, **kwargs):
97
    return _RLock(*args, **kwargs)
98
99
class _RLock(_Verbose):
100
101
    def __init__(self, verbose=None):
102
        _Verbose.__init__(self, verbose)
103
        self.__block = _allocate_lock()
104
        self.__owner = None
105
        self.__count = 0
106
107
    def __repr__(self):
108
        owner = self.__owner
109
        return "<%s(%s, %d)>" % (
110
                self.__class__.__name__,
111
                owner and owner.name,
112
                self.__count)
113
114
    def acquire(self, blocking=1):
115
        me = current_thread()
116
        if self.__owner is me:
117
            self.__count = self.__count + 1
118
            if __debug__:
119
                self._note("%s.acquire(%s): recursive success", self, blocking)
120
            return 1
121
        rc = self.__block.acquire(blocking)
122
        if rc:
123
            self.__owner = me
124
            self.__count = 1
125
            if __debug__:
126
                self._note("%s.acquire(%s): initial success", self, blocking)
127
        else:
128
            if __debug__:
129
                self._note("%s.acquire(%s): failure", self, blocking)
130
        return rc
131
132
    __enter__ = acquire
133
134
    def release(self):
135
        if self.__owner is not current_thread():
136
            raise RuntimeError("cannot release un-aquired lock")
137
        self.__count = count = self.__count - 1
138
        if not count:
139
            self.__owner = None
140
            self.__block.release()
141
            if __debug__:
142
                self._note("%s.release(): final release", self)
143
        else:
144
            if __debug__:
145
                self._note("%s.release(): non-final release", self)
146
147
    def __exit__(self, t, v, tb):
148
        self.release()
149
150
    # Internal methods used by condition variables
151
152
    def _acquire_restore(self, count_owner):
153
        count, owner = count_owner
154
        self.__block.acquire()
155
        self.__count = count
156
        self.__owner = owner
157
        if __debug__:
158
            self._note("%s._acquire_restore()", self)
159
160
    def _release_save(self):
161
        if __debug__:
162
            self._note("%s._release_save()", self)
163
        count = self.__count
164
        self.__count = 0
165
        owner = self.__owner
166
        self.__owner = None
167
        self.__block.release()
168
        return (count, owner)
169
170
    def _is_owned(self):
171
        return self.__owner is current_thread()
172
173
0.000223 sec 174
def Condition(*args, **kwargs):
3.8e-05 sec 175
    return _Condition(*args, **kwargs)
176
177
class _Condition(_Verbose):
178
0.000146 sec 179
    def __init__(self, lock=None, verbose=None):
4.2e-05 sec 180
        _Verbose.__init__(self, verbose)
5e-05 sec 181
        if lock is None:
182
            lock = RLock()
4.7e-05 sec 183
        self.__lock = lock
184
        # Export the lock's acquire() and release() methods
4.9e-05 sec 185
        self.acquire = lock.acquire
6.3e-05 sec 186
        self.release = lock.release
187
        # If the lock defines _release_save() and/or _acquire_restore(),
188
        # these override the default implementations (which just call
189
        # release() and acquire() on the lock).  Ditto for _is_owned().
5.8e-05 sec 190
        try:
4.2e-05 sec 191
            self._release_save = lock._release_save
0.00015 sec 192
        except AttributeError:
0.007759 sec 193
            pass
4.3e-05 sec 194
        try:
4.2e-05 sec 195
            self._acquire_restore = lock._acquire_restore
0.000142 sec 196
        except AttributeError:
6.9e-05 sec 197
            pass
4e-05 sec 198
        try:
4.3e-05 sec 199
            self._is_owned = lock._is_owned
0.000121 sec 200
        except AttributeError:
6.7e-05 sec 201
            pass
4.1e-05 sec 202
        self.__waiters = []
203
204
    def __enter__(self):
205
        return self.__lock.__enter__()
206
207
    def __exit__(self, *args):
208
        return self.__lock.__exit__(*args)
209
210
    def __repr__(self):
211
        return "<Condition(%s, %d)>" % (self.__lock, len(self.__waiters))
212
0.000106 sec 213
    def _release_save(self):
2.4e-05 sec 214
        self.__lock.release()           # No state to save
215
0.000121 sec 216
    def _acquire_restore(self, x):
2.3e-05 sec 217
        self.__lock.acquire()           # Ignore saved state
218
0.000143 sec 219
    def _is_owned(self):
220
        # Return True if lock is owned by current_thread.
221
        # This method is called only if __lock doesn't have _is_owned().
0.00023 sec 222
        if self.__lock.acquire(0):
223
            self.__lock.release()
224
            return False
225
        else:
0.000169 sec 226
            return True
227
8.3e-05 sec 228
    def wait(self, timeout=None):
2.3e-05 sec 229
        if not self._is_owned():
230
            raise RuntimeError("cannot wait on un-aquired lock")
3.1e-05 sec 231
        waiter = _allocate_lock()
5e-05 sec 232
        waiter.acquire()
8.3e-05 sec 233
        self.__waiters.append(waiter)
5.5e-05 sec 234
        saved_state = self._release_save()
2.5e-05 sec 235
        try:    # restore state no matter what (e.g., KeyboardInterrupt)
2.3e-05 sec 236
            if timeout is None:
2.2e-05 sec 237
                waiter.acquire()
238
                if __debug__:
0.001651 sec 239
                    self._note("%s.wait(): got it", self)
240
            else:
241
                # Balancing act:  We can't afford a pure busy loop, so we
242
                # have to sleep; but if we sleep the whole timeout time,
243
                # we'll be unresponsive.  The scheme here sleeps very
244
                # little at first, longer as time goes on, but never longer
245
                # than 20 times per second (or the timeout time remaining).
5e-06 sec 246
                endtime = _time() + timeout
8e-06 sec 247
                delay = 0.0005 # 500 us -> initial delay of 1 ms
4.999032 sec 248
                while True:
0.000254 sec 249
                    gotit = waiter.acquire(0)
0.000435 sec 250
                    if gotit:
2e-06 sec 251
                        break
0.000219 sec 252
                    remaining = endtime - _time()
0.000487 sec 253
                    if remaining <= 0:
3e-06 sec 254
                        break
0.000294 sec 255
                    delay = min(delay * 2, remaining, .05)
0.000477 sec 256
                    _sleep(delay)
5e-06 sec 257
                if not gotit:
258
                    if __debug__:
2e-06 sec 259
                        self._note("%s.wait(%s): timed out", self, timeout)
3e-06 sec 260
                    try:
2e-06 sec 261
                        self.__waiters.remove(waiter)
262
                    except ValueError:
263
                        pass
264
                else:
265
                    if __debug__:
3e-06 sec 266
                        self._note("%s.wait(%s): got it", self, timeout)
267
        finally:
4.8e-05 sec 268
            self._acquire_restore(saved_state)
269
7.1e-05 sec 270
    def notify(self, n=1):
1.7e-05 sec 271
        if not self._is_owned():
272
            raise RuntimeError("cannot notify on un-aquired lock")
2.2e-05 sec 273
        __waiters = self.__waiters
2.5e-05 sec 274
        waiters = __waiters[:n]
3e-05 sec 275
        if not waiters:
276
            if __debug__:
2e-06 sec 277
                self._note("%s.notify(): no waiters", self)
2e-06 sec 278
            return
2.2e-05 sec 279
        self._note("%s.notify(): notifying %d waiter%s", self, n,
2.6e-05 sec 280
                   n!=1 and "s" or "")
6.1e-05 sec 281
        for waiter in waiters:
2.6e-05 sec 282
            waiter.release()
7.9e-05 sec 283
            try:
1.8e-05 sec 284
                __waiters.remove(waiter)
285
            except ValueError:
286
                pass
287
288
    def notifyAll(self):
289
        self.notify(len(self.__waiters))
290
291
    notify_all = notifyAll
292
293
294
def Semaphore(*args, **kwargs):
295
    return _Semaphore(*args, **kwargs)
296
297
class _Semaphore(_Verbose):
298
299
    # After Tim Peters' semaphore class, but not quite the same (no maximum)
300
301
    def __init__(self, value=1, verbose=None):
302
        if value < 0:
303
            raise ValueError("semaphore initial value must be >= 0")
304
        _Verbose.__init__(self, verbose)
305
        self.__cond = Condition(Lock())
306
        self.__value = value
307
308
    def acquire(self, blocking=1):
309
        rc = False
310
        self.__cond.acquire()
311
        while self.__value == 0:
312
            if not blocking:
313
                break
314
            if __debug__:
315
                self._note("%s.acquire(%s): blocked waiting, value=%s",
316
                           self, blocking, self.__value)
317
            self.__cond.wait()
318
        else:
319
            self.__value = self.__value - 1
320
            if __debug__:
321
                self._note("%s.acquire: success, value=%s",
322
                           self, self.__value)
323
            rc = True
324
        self.__cond.release()
325
        return rc
326
327
    __enter__ = acquire
328
329
    def release(self):
330
        self.__cond.acquire()
331
        self.__value = self.__value + 1
332
        if __debug__:
333
            self._note("%s.release: success, value=%s",
334
                       self, self.__value)
335
        self.__cond.notify()
336
        self.__cond.release()
337
338
    def __exit__(self, t, v, tb):
339
        self.release()
340
341
342
def BoundedSemaphore(*args, **kwargs):
343
    return _BoundedSemaphore(*args, **kwargs)
344
345
class _BoundedSemaphore(_Semaphore):
346
    """Semaphore that checks that # releases is <= # acquires"""
347
    def __init__(self, value=1, verbose=None):
348
        _Semaphore.__init__(self, value, verbose)
349
        self._initial_value = value
350
351
    def release(self):
352
        if self._Semaphore__value >= self._initial_value:
353
            raise ValueError, "Semaphore released too many times"
354
        return _Semaphore.release(self)
355
356
5.2e-05 sec 357
def Event(*args, **kwargs):
1.9e-05 sec 358
    return _Event(*args, **kwargs)
359
360
class _Event(_Verbose):
361
362
    # After Tim Peters' event class (without is_posted())
363
6.4e-05 sec 364
    def __init__(self, verbose=None):
1.9e-05 sec 365
        _Verbose.__init__(self, verbose)
2.1e-05 sec 366
        self.__cond = Condition(Lock())
2.8e-05 sec 367
        self.__flag = False
368
0.000142 sec 369
    def isSet(self):
4.3e-05 sec 370
        return self.__flag
371
372
    is_set = isSet
373
374
    def set(self):
375
        self.__cond.acquire()
376
        try:
377
            self.__flag = True
378
            self.__cond.notify_all()
379
        finally:
380
            self.__cond.release()
381
382
    def clear(self):
383
        self.__cond.acquire()
384
        try:
385
            self.__flag = False
386
        finally:
387
            self.__cond.release()
388
0.00011 sec 389
    def wait(self, timeout=None):
1.6e-05 sec 390
        self.__cond.acquire()
0.000171 sec 391
        try:
2.1e-05 sec 392
            if not self.__flag:
2e-05 sec 393
                self.__cond.wait(timeout)
394
        finally:
4.4e-05 sec 395
            self.__cond.release()
396
397
# Helper to generate new thread names
398
_counter = 0
0.000144 sec 399
def _newname(template="Thread-%d"):
400
    global _counter
2.1e-05 sec 401
    _counter = _counter + 1
2.3e-05 sec 402
    return template % _counter
403
404
# Active thread administration
405
_active_limbo_lock = _allocate_lock()
406
_active = {}    # maps thread id to Thread object
407
_limbo = {}
408
409
410
# Main class for threads
411
412
class Thread(_Verbose):
413
414
    __initialized = False
415
    # Need to store a reference to sys.exc_info for printing
416
    # out exceptions when a thread tries to use a global var. during interp.
417
    # shutdown and thus raises an exception about trying to perform some
418
    # operation on/with a NoneType
419
    __exc_info = _sys.exc_info
420
    # Keep sys.exc_clear too to clear the exception just before
421
    # allowing .join() to return.
422
    __exc_clear = _sys.exc_clear
423
8e-05 sec 424
    def __init__(self, group=None, target=None, name=None,
425
                 args=(), kwargs=None, verbose=None):
1.7e-05 sec 426
        assert group is None, "group argument must be None for now"
2e-05 sec 427
        _Verbose.__init__(self, verbose)
2.4e-05 sec 428
        if kwargs is None:
2e-05 sec 429
            kwargs = {}
2.1e-05 sec 430
        self.__target = target
2.5e-05 sec 431
        self.__name = str(name or _newname())
4.4e-05 sec 432
        self.__args = args
4.8e-05 sec 433
        self.__kwargs = kwargs
2.2e-05 sec 434
        self.__daemonic = self._set_daemon()
2.2e-05 sec 435
        self.__ident = None
2.5e-05 sec 436
        self.__started = Event()
2.6e-05 sec 437
        self.__stopped = False
2.3e-05 sec 438
        self.__block = Condition(Lock())
2.8e-05 sec 439
        self.__initialized = True
440
        # sys.stderr is not stored in the class like
441
        # sys.exc_info since it can be changed between instances
2.9e-05 sec 442
        self.__stderr = _sys.stderr
443
5.6e-05 sec 444
    def _set_daemon(self):
445
        # Overridden in _MainThread and _DummyThread
1.9e-05 sec 446
        return current_thread().daemon
447
448
    def __repr__(self):
449
        assert self.__initialized, "Thread.__init__() was not called"
450
        status = "initial"
451
        if self.__started.is_set():
452
            status = "started"
453
        if self.__stopped:
454
            status = "stopped"
455
        if self.__daemonic:
456
            status += " daemon"
457
        if self.__ident is not None:
458
            status += " %s" % self.__ident
459
        return "<%s(%s, %s)>" % (self.__class__.__name__, self.__name, status)
460
5.4e-05 sec 461
    def start(self):
1.9e-05 sec 462
        if not self.__initialized:
463
            raise RuntimeError("thread.__init__() not called")
2.6e-05 sec 464
        if self.__started.is_set():
465
            raise RuntimeError("thread already started")
466
        if __debug__:
2.2e-05 sec 467
            self._note("%s.start(): starting thread", self)
2.1e-05 sec 468
        _active_limbo_lock.acquire()
0.000129 sec 469
        _limbo[self] = self
3.6e-05 sec 470
        _active_limbo_lock.release()
2.6e-05 sec 471
        _start_new_thread(self.__bootstrap, ())
0.000634 sec 472
        self.__started.wait()
473
474
    def run(self):
475
        try:
476
            if self.__target:
477
                self.__target(*self.__args, **self.__kwargs)
478
        finally:
479
            # Avoid a refcycle if the thread is running a function with
480
            # an argument that has a member that points to the thread.
481
            del self.__target, self.__args, self.__kwargs
482
483
    def __bootstrap(self):
484
        # Wrapper around the real bootstrap code that ignores
485
        # exceptions during interpreter cleanup.  Those typically
486
        # happen when a daemon thread wakes up at an unfortunate
487
        # moment, finds the world around it destroyed, and raises some
488
        # random exception *** while trying to report the exception in
489
        # __bootstrap_inner() below ***.  Those random exceptions
490
        # don't help anybody, and they confuse users, so we suppress
491
        # them.  We suppress them only when it appears that the world
492
        # indeed has already been destroyed, so that exceptions in
493
        # __bootstrap_inner() during normal business hours are properly
494
        # reported.  Also, we only suppress them for daemonic threads;
495
        # if a non-daemonic encounters this, something else is wrong.
496
        try:
497
            self.__bootstrap_inner()
498
        except:
499
            if self.__daemonic and _sys is None:
500
                return
501
            raise
502
503
    def _set_ident(self):
504
        self.__ident = _get_ident()
505
506
    def __bootstrap_inner(self):
507
        try:
508
            self._set_ident()
509
            self.__started.set()
510
            _active_limbo_lock.acquire()
511
            _active[self.__ident] = self
512
            del _limbo[self]
513
            _active_limbo_lock.release()
514
            if __debug__:
515
                self._note("%s.__bootstrap(): thread started", self)
516
517
            if _trace_hook:
518
                self._note("%s.__bootstrap(): registering trace hook", self)
519
                _sys.settrace(_trace_hook)
520
            if _profile_hook:
521
                self._note("%s.__bootstrap(): registering profile hook", self)
522
                _sys.setprofile(_profile_hook)
523
524
            try:
525
                self.run()
526
            except SystemExit:
527
                if __debug__:
528
                    self._note("%s.__bootstrap(): raised SystemExit", self)
529
            except:
530
                if __debug__:
531
                    self._note("%s.__bootstrap(): unhandled exception", self)
532
                # If sys.stderr is no more (most likely from interpreter
533
                # shutdown) use self.__stderr.  Otherwise still use sys (as in
534
                # _sys) in case sys.stderr was redefined since the creation of
535
                # self.
536
                if _sys:
537
                    _sys.stderr.write("Exception in thread %s:\n%s\n" %
538
                                      (self.name, _format_exc()))
539
                else:
540
                    # Do the best job possible w/o a huge amt. of code to
541
                    # approximate a traceback (code ideas from
542
                    # Lib/traceback.py)
543
                    exc_type, exc_value, exc_tb = self.__exc_info()
544
                    try:
545
                        print>>self.__stderr, (
546
                            "Exception in thread " + self.name +
547
                            " (most likely raised during interpreter shutdown):")
548
                        print>>self.__stderr, (
549
                            "Traceback (most recent call last):")
550
                        while exc_tb:
551
                            print>>self.__stderr, (
552
                                '  File "%s", line %s, in %s' %
553
                                (exc_tb.tb_frame.f_code.co_filename,
554
                                    exc_tb.tb_lineno,
555
                                    exc_tb.tb_frame.f_code.co_name))
556
                            exc_tb = exc_tb.tb_next
557
                        print>>self.__stderr, ("%s: %s" % (exc_type, exc_value))
558
                    # Make sure that exc_tb gets deleted since it is a memory
559
                    # hog; deleting everything else is just for thoroughness
560
                    finally:
561
                        del exc_type, exc_value, exc_tb
562
            else:
563
                if __debug__:
564
                    self._note("%s.__bootstrap(): normal return", self)
565
            finally:
566
                # Prevent a race in
567
                # test_threading.test_no_refcycle_through_target when
568
                # the exception keeps the target alive past when we
569
                # assert that it's dead.
570
                self.__exc_clear()
571
        finally:
572
            with _active_limbo_lock:
573
                self.__stop()
574
                try:
575
                    # We don't call self.__delete() because it also
576
                    # grabs _active_limbo_lock.
577
                    del _active[_get_ident()]
578
                except:
579
                    pass
580
581
    def __stop(self):
582
        self.__block.acquire()
583
        self.__stopped = True
584
        self.__block.notify_all()
585
        self.__block.release()
586
587
    def __delete(self):
588
        "Remove current thread from the dict of currently running threads."
589
590
        # Notes about running with dummy_thread:
591
        #
592
        # Must take care to not raise an exception if dummy_thread is being
593
        # used (and thus this module is being used as an instance of
594
        # dummy_threading).  dummy_thread.get_ident() always returns -1 since
595
        # there is only one thread if dummy_thread is being used.  Thus
596
        # len(_active) is always <= 1 here, and any Thread instance created
597
        # overwrites the (if any) thread currently registered in _active.
598
        #
599
        # An instance of _MainThread is always created by 'threading'.  This
600
        # gets overwritten the instant an instance of Thread is created; both
601
        # threads return -1 from dummy_thread.get_ident() and thus have the
602
        # same key in the dict.  So when the _MainThread instance created by
603
        # 'threading' tries to clean itself up when atexit calls this method
604
        # it gets a KeyError if another Thread instance was created.
605
        #
606
        # This all means that KeyError from trying to delete something from
607
        # _active if dummy_threading is being used is a red herring.  But
608
        # since it isn't if dummy_threading is *not* being used then don't
609
        # hide the exception.
610
611
        try:
612
            with _active_limbo_lock:
613
                del _active[_get_ident()]
614
                # There must not be any python code between the previous line
615
                # and after the lock is released.  Otherwise a tracing function
616
                # could try to acquire the lock again in the same thread, (in
617
                # current_thread()), and would block.
618
        except KeyError:
619
            if 'dummy_threading' not in _sys.modules:
620
                raise
621
2.4e-05 sec 622
    def join(self, timeout=None):
5e-06 sec 623
        if not self.__initialized:
624
            raise RuntimeError("Thread.__init__() not called")
8e-06 sec 625
        if not self.__started.is_set():
626
            raise RuntimeError("cannot join thread before it is started")
6e-06 sec 627
        if self is current_thread():
628
            raise RuntimeError("cannot join current thread")
629
630
        if __debug__:
5e-06 sec 631
            if not self.__stopped:
6e-06 sec 632
                self._note("%s.join(): waiting until thread stops", self)
7e-06 sec 633
        self.__block.acquire()
1.2e-05 sec 634
        try:
5e-06 sec 635
            if timeout is None:
5e-06 sec 636
                while not self.__stopped:
2e-06 sec 637
                    self.__block.wait()
638
                if __debug__:
3e-06 sec 639
                    self._note("%s.join(): thread stopped", self)
640
            else:
5e-06 sec 641
                deadline = _time() + timeout
2.5e-05 sec 642
                while not self.__stopped:
8e-06 sec 643
                    delay = deadline - _time()
1.4e-05 sec 644
                    if delay <= 0:
645
                        if __debug__:
3e-06 sec 646
                            self._note("%s.join(): timed out", self)
2e-06 sec 647
                        break
7e-06 sec 648
                    self.__block.wait(delay)
649
                else:
650
                    if __debug__:
3e-06 sec 651
                        self._note("%s.join(): thread stopped", self)
652
        finally:
7e-06 sec 653
            self.__block.release()
654
5.9e-05 sec 655
    @property
656
    def name(self):
1.8e-05 sec 657
        assert self.__initialized, "Thread.__init__() not called"
2.4e-05 sec 658
        return self.__name
659
8.8e-05 sec 660
    @name.setter
661
    def name(self, name):
1.8e-05 sec 662
        assert self.__initialized, "Thread.__init__() not called"
2.2e-05 sec 663
        self.__name = str(name)
664
665
    @property
666
    def ident(self):
667
        assert self.__initialized, "Thread.__init__() not called"
668
        return self.__ident
669
7.6e-05 sec 670
    def isAlive(self):
2.3e-05 sec 671
        assert self.__initialized, "Thread.__init__() not called"
3e-05 sec 672
        return self.__started.is_set() and not self.__stopped
673
674
    is_alive = isAlive
675
6.2e-05 sec 676
    @property
677
    def daemon(self):
1.9e-05 sec 678
        assert self.__initialized, "Thread.__init__() not called"
2.3e-05 sec 679
        return self.__daemonic
680
681
    @daemon.setter
682
    def daemon(self, daemonic):
683
        if not self.__initialized:
684
            raise RuntimeError("Thread.__init__() not called")
685
        if self.__started.is_set():
686
            raise RuntimeError("cannot set daemon status of active thread");
687
        self.__daemonic = daemonic
688
689
    def isDaemon(self):
690
        return self.daemon
691
692
    def setDaemon(self, daemonic):
693
        self.daemon = daemonic
694
7.5e-05 sec 695
    def getName(self):
1.8e-05 sec 696
        return self.name
697
6.1e-05 sec 698
    def setName(self, name):
1.8e-05 sec 699
        self.name = name
700
701
# The timer class was contributed by Itamar Shtull-Trauring
702
703
def Timer(*args, **kwargs):
704
    return _Timer(*args, **kwargs)
705
706
class _Timer(Thread):
707
    """Call a function after a specified number of seconds:
708
709
    t = Timer(30.0, f, args=[], kwargs={})
710
    t.start()
711
    t.cancel() # stop the timer's action if it's still waiting
712
    """
713
714
    def __init__(self, interval, function, args=[], kwargs={}):
715
        Thread.__init__(self)
716
        self.interval = interval
717
        self.function = function
718
        self.args = args
719
        self.kwargs = kwargs
720
        self.finished = Event()
721
722
    def cancel(self):
723
        """Stop the timer if it hasn't finished yet"""
724
        self.finished.set()
725
726
    def run(self):
727
        self.finished.wait(self.interval)
728
        if not self.finished.is_set():
729
            self.function(*self.args, **self.kwargs)
730
        self.finished.set()
731
732
# Special thread class to represent the main thread
733
# This is garbage collected through an exit handler
734
735
class _MainThread(Thread):
736
737
    def __init__(self):
738
        Thread.__init__(self, name="MainThread")
739
        self._Thread__started.set()
740
        self._set_ident()
741
        _active_limbo_lock.acquire()
742
        _active[_get_ident()] = self
743
        _active_limbo_lock.release()
744
745
    def _set_daemon(self):
746
        return False
747
748
    def _exitfunc(self):
749
        self._Thread__stop()
750
        t = _pickSomeNonDaemonThread()
751
        if t:
752
            if __debug__:
753
                self._note("%s: waiting for other threads", self)
754
        while t:
755
            t.join()
756
            t = _pickSomeNonDaemonThread()
757
        if __debug__:
758
            self._note("%s: exiting", self)
759
        self._Thread__delete()
760
761
def _pickSomeNonDaemonThread():
762
    for t in enumerate():
763
        if not t.daemon and t.is_alive():
764
            return t
765
    return None
766
767
768
# Dummy thread class to represent threads not started here.
769
# These aren't garbage collected when they die, nor can they be waited for.
770
# If they invoke anything in threading.py that calls current_thread(), they
771
# leave an entry in the _active dict forever after.
772
# Their purpose is to return *something* from current_thread().
773
# They are marked as daemon threads so we won't wait for them
774
# when we exit (conform previous semantics).
775
776
class _DummyThread(Thread):
777
778
    def __init__(self):
779
        Thread.__init__(self, name=_newname("Dummy-%d"))
780
781
        # Thread.__block consumes an OS-level locking primitive, which
782
        # can never be used by a _DummyThread.  Since a _DummyThread
783
        # instance is immortal, that's bad, so release this resource.
784
        del self._Thread__block
785
786
        self._Thread__started.set()
787
        self._set_ident()
788
        _active_limbo_lock.acquire()
789
        _active[_get_ident()] = self
790
        _active_limbo_lock.release()
791
792
    def _set_daemon(self):
793
        return True
794
795
    def join(self, timeout=None):
796
        assert False, "cannot join a dummy thread"
797
798
799
# Global API functions
800
9.2e-05 sec 801
def currentThread():
2.3e-05 sec 802
    try:
2.5e-05 sec 803
        return _active[_get_ident()]
804
    except KeyError:
805
        ##print "current_thread(): no current thread for", _get_ident()
806
        return _DummyThread()
807
808
current_thread = currentThread
809
810
def activeCount():
811
    _active_limbo_lock.acquire()
812
    count = len(_active) + len(_limbo)
813
    _active_limbo_lock.release()
814
    return count
815
816
active_count = activeCount
817
818
def enumerate():
819
    _active_limbo_lock.acquire()
820
    active = _active.values() + _limbo.values()
821
    _active_limbo_lock.release()
822
    return active
823
824
from thread import stack_size
825
826
# Create the main thread object,
827
# and make it available for the interpreter
828
# (Py_Main) as threading._shutdown.
829
830
_shutdown = _MainThread()._exitfunc
831
832
# get thread-local implementation, either from the thread
833
# module, or from the python fallback
834
835
try:
836
    from thread import _local as local
837
except ImportError:
838
    from _threading_local import local
839
840
841
def _after_fork():
842
    # This function is called by Python/ceval.c:PyEval_ReInitThreads which
843
    # is called from PyOS_AfterFork.  Here we cleanup threading module state
844
    # that should not exist after a fork.
845
846
    # Reset _active_limbo_lock, in case we forked while the lock was held
847
    # by another (non-forked) thread.  http://bugs.python.org/issue874900
848
    global _active_limbo_lock
849
    _active_limbo_lock = _allocate_lock()
850
851
    # fork() only copied the current thread; clear references to others.
852
    new_active = {}
853
    current = current_thread()
854
    with _active_limbo_lock:
855
        for thread in _active.itervalues():
856
            if thread is current:
857
                # There is only one active thread. We reset the ident to
858
                # its new value since it can have changed.
859
                ident = _get_ident()
860
                thread._Thread__ident = ident
861
                new_active[ident] = thread
862
            else:
863
                # All the others are already stopped.
864
                # We don't call _Thread__stop() because it tries to acquire
865
                # thread._Thread__block which could also have been held while
866
                # we forked.
867
                thread._Thread__stopped = True
868
869
        _limbo.clear()
870
        _active.clear()
871
        _active.update(new_active)
872
        assert len(_active) == 1
873
874
875
# Self-test code
876
877
def _test():
878
879
    class BoundedQueue(_Verbose):
880
881
        def __init__(self, limit):
882
            _Verbose.__init__(self)
883
            self.mon = RLock()
884
            self.rc = Condition(self.mon)
885
            self.wc = Condition(self.mon)
886
            self.limit = limit
887
            self.queue = deque()
888
889
        def put(self, item):
890
            self.mon.acquire()
891
            while len(self.queue) >= self.limit:
892
                self._note("put(%s): queue full", item)
893
                self.wc.wait()
894
            self.queue.append(item)
895
            self._note("put(%s): appended, length now %d",
896
                       item, len(self.queue))
897
            self.rc.notify()
898
            self.mon.release()
899
900
        def get(self):
901
            self.mon.acquire()
902
            while not self.queue:
903
                self._note("get(): queue empty")
904
                self.rc.wait()
905
            item = self.queue.popleft()
906
            self._note("get(): got %s, %d left", item, len(self.queue))
907
            self.wc.notify()
908
            self.mon.release()
909
            return item
910
911
    class ProducerThread(Thread):
912
913
        def __init__(self, queue, quota):
914
            Thread.__init__(self, name="Producer")
915
            self.queue = queue
916
            self.quota = quota
917
918
        def run(self):
919
            from random import random
920
            counter = 0
921
            while counter < self.quota:
922
                counter = counter + 1
923
                self.queue.put("%s.%d" % (self.name, counter))
924
                _sleep(random() * 0.00001)
925
926
927
    class ConsumerThread(Thread):
928
929
        def __init__(self, queue, count):
930
            Thread.__init__(self, name="Consumer")
931
            self.queue = queue
932
            self.count = count
933
934
        def run(self):
935
            while self.count > 0:
936
                item = self.queue.get()
937
                print item
938
                self.count = self.count - 1
939
940
    NP = 3
941
    QL = 4
942
    NI = 5
943
944
    Q = BoundedQueue(QL)
945
    P = []
946
    for i in range(NP):
947
        t = ProducerThread(Q, NI)
948
        t.name = ("Producer-%d" % (i+1))
949
        P.append(t)
950
    C = ConsumerThread(Q, NI*NP)
951
    for t in P:
952
        t.start()
953
        _sleep(0.000001)
954
    C.start()
955
    for t in P:
956
        t.join()
957
    C.join()
958
959
if __name__ == '__main__':
960
    _test()
/usr/lib/python2.6/warnings.py
     time      num code
1
"""Python part of the warnings subsystem."""
2
3
# Note: function level imports should *not* be used
4
# in this module as it may cause import lock deadlock.
5
# See bug 683658.
6
import linecache
7
import sys
8
import types
9
10
__all__ = ["warn", "showwarning", "formatwarning", "filterwarnings",
11
           "resetwarnings", "catch_warnings"]
12
13
14
def warnpy3k(message, category=None, stacklevel=1):
15
    """Issue a deprecation warning for Python 3.x related changes.
16
17
    Warnings are omitted unless Python is started with the -3 option.
18
    """
19
    if sys.py3kwarning:
20
        if category is None:
21
            category = DeprecationWarning
22
        warn(message, category, stacklevel+1)
23
24
def _show_warning(message, category, filename, lineno, file=None, line=None):
25
    """Hook to write a warning to a file; replace if you like."""
26
    if file is None:
27
        file = sys.stderr
28
    try:
29
        file.write(formatwarning(message, category, filename, lineno, line))
30
    except IOError:
31
        pass # the file (probably stderr) is invalid - this warning gets lost.
32
# Keep a worrking version around in case the deprecation of the old API is
33
# triggered.
34
showwarning = _show_warning
35
36
def formatwarning(message, category, filename, lineno, line=None):
37
    """Function to format a warning the standard way."""
38
    s =  "%s:%s: %s: %s\n" % (filename, lineno, category.__name__, message)
39
    line = linecache.getline(filename, lineno) if line is None else line
40
    if line:
41
        line = line.strip()
42
        s += "  %s\n" % line
43
    return s
44
45
def filterwarnings(action, message="", category=Warning, module="", lineno=0,
46
                   append=0):
47
    """Insert an entry into the list of warnings filters (at the front).
48
49
    Use assertions to check that all arguments have the right type."""
50
    import re
51
    assert action in ("error", "ignore", "always", "default", "module",
52
                      "once"), "invalid action: %r" % (action,)
53
    assert isinstance(message, basestring), "message must be a string"
54
    assert isinstance(category, (type, types.ClassType)), \
55
           "category must be a class"
56
    assert issubclass(category, Warning), "category must be a Warning subclass"
57
    assert isinstance(module, basestring), "module must be a string"
58
    assert isinstance(lineno, int) and lineno >= 0, \
59
           "lineno must be an int >= 0"
60
    item = (action, re.compile(message, re.I), category,
61
            re.compile(module), lineno)
62
    if append:
63
        filters.append(item)
64
    else:
65
        filters.insert(0, item)
66
67
def simplefilter(action, category=Warning, lineno=0, append=0):
68
    """Insert a simple entry into the list of warnings filters (at the front).
69
70
    A simple filter matches all modules and messages.
71
    """
72
    assert action in ("error", "ignore", "always", "default", "module",
73
                      "once"), "invalid action: %r" % (action,)
74
    assert isinstance(lineno, int) and lineno >= 0, \
75
           "lineno must be an int >= 0"
76
    item = (action, None, category, None, lineno)
77
    if append:
78
        filters.append(item)
79
    else:
80
        filters.insert(0, item)
81
82
def resetwarnings():
83
    """Clear the list of warning filters, so that no filters are active."""
84
    filters[:] = []
85
86
class _OptionError(Exception):
87
    """Exception used by option processing helpers."""
88
    pass
89
90
# Helper to process -W options passed via sys.warnoptions
91
def _processoptions(args):
92
    for arg in args:
93
        try:
94
            _setoption(arg)
95
        except _OptionError, msg:
96
            print >>sys.stderr, "Invalid -W option ignored:", msg
97
98
# Helper for _processoptions()
99
def _setoption(arg):
100
    import re
101
    parts = arg.split(':')
102
    if len(parts) > 5:
103
        raise _OptionError("too many fields (max 5): %r" % (arg,))
104
    while len(parts) < 5:
105
        parts.append('')
106
    action, message, category, module, lineno = [s.strip()
107
                                                 for s in parts]
108
    action = _getaction(action)
109
    message = re.escape(message)
110
    category = _getcategory(category)
111
    module = re.escape(module)
112
    if module:
113
        module = module + '$'
114
    if lineno:
115
        try:
116
            lineno = int(lineno)
117
            if lineno < 0:
118
                raise ValueError
119
        except (ValueError, OverflowError):
120
            raise _OptionError("invalid lineno %r" % (lineno,))
121
    else:
122
        lineno = 0
123
    filterwarnings(action, message, category, module, lineno)
124
125
# Helper for _setoption()
126
def _getaction(action):
127
    if not action:
128
        return "default"
129
    if action == "all": return "always" # Alias
130
    for a in ('default', 'always', 'ignore', 'module', 'once', 'error'):
131
        if a.startswith(action):
132
            return a
133
    raise _OptionError("invalid action: %r" % (action,))
134
135
# Helper for _setoption()
136
def _getcategory(category):
137
    import re
138
    if not category:
139
        return Warning
140
    if re.match("^[a-zA-Z0-9_]+$", category):
141
        try:
142
            cat = eval(category)
143
        except NameError:
144
            raise _OptionError("unknown warning category: %r" % (category,))
145
    else:
146
        i = category.rfind(".")
147
        module = category[:i]
148
        klass = category[i+1:]
149
        try:
150
            m = __import__(module, None, None, [klass])
151
        except ImportError:
152
            raise _OptionError("invalid module name: %r" % (module,))
153
        try:
154
            cat = getattr(m, klass)
155
        except AttributeError:
156
            raise _OptionError("unknown warning category: %r" % (category,))
157
    if not issubclass(cat, Warning):
158
        raise _OptionError("invalid warning category: %r" % (category,))
159
    return cat
160
161
162
# Code typically replaced by _warnings
163
def warn(message, category=None, stacklevel=1):
164
    """Issue a warning, or maybe ignore it or raise an exception."""
165
    # Check if message is already a Warning object
166
    if isinstance(message, Warning):
167
        category = message.__class__
168
    # Check category argument
169
    if category is None:
170
        category = UserWarning
171
    assert issubclass(category, Warning)
172
    # Get context information
173
    try:
174
        caller = sys._getframe(stacklevel)
175
    except ValueError:
176
        globals = sys.__dict__
177
        lineno = 1
178
    else:
179
        globals = caller.f_globals
180
        lineno = caller.f_lineno
181
    if '__name__' in globals:
182
        module = globals['__name__']
183
    else:
184
        module = "<string>"
185
    filename = globals.get('__file__')
186
    if filename:
187
        fnl = filename.lower()
188
        if fnl.endswith((".pyc", ".pyo")):
189
            filename = filename[:-1]
190
    else:
191
        if module == "__main__":
192
            try:
193
                filename = sys.argv[0]
194
            except AttributeError:
195
                # embedded interpreters don't have sys.argv, see bug #839151
196
                filename = '__main__'
197
        if not filename:
198
            filename = module
199
    registry = globals.setdefault("__warningregistry__", {})
200
    warn_explicit(message, category, filename, lineno, module, registry,
201
                  globals)
202
203
def warn_explicit(message, category, filename, lineno,
204
                  module=None, registry=None, module_globals=None):
205
    lineno = int(lineno)
206
    if module is None:
207
        module = filename or "<unknown>"
208
        if module[-3:].lower() == ".py":
209
            module = module[:-3] # XXX What about leading pathname?
210
    if registry is None:
211
        registry = {}
212
    if isinstance(message, Warning):
213
        text = str(message)
214
        category = message.__class__
215
    else:
216
        text = message
217
        message = category(message)
218
    key = (text, category, lineno)
219
    # Quick test for common case
220
    if registry.get(key):
221
        return
222
    # Search the filters
223
    for item in filters:
224
        action, msg, cat, mod, ln = item
225
        if ((msg is None or msg.match(text)) and
226
            issubclass(category, cat) and
227
            (mod is None or mod.match(module)) and
228
            (ln == 0 or lineno == ln)):
229
            break
230
    else:
231
        action = defaultaction
232
    # Early exit actions
233
    if action == "ignore":
234
        registry[key] = 1
235
        return
236
237
    # Prime the linecache for formatting, in case the
238
    # "file" is actually in a zipfile or something.
239
    linecache.getlines(filename, module_globals)
240
241
    if action == "error":
242
        raise message
243
    # Other actions
244
    if action == "once":
245
        registry[key] = 1
246
        oncekey = (text, category)
247
        if onceregistry.get(oncekey):
248
            return
249
        onceregistry[oncekey] = 1
250
    elif action == "always":
251
        pass
252
    elif action == "module":
253
        registry[key] = 1
254
        altkey = (text, category, 0)
255
        if registry.get(altkey):
256
            return
257
        registry[altkey] = 1
258
    elif action == "default":
259
        registry[key] = 1
260
    else:
261
        # Unrecognized actions are errors
262
        raise RuntimeError(
263
              "Unrecognized action (%r) in warnings.filters:\n %s" %
264
              (action, item))
265
    # Warn if showwarning() does not support the 'line' argument.
266
    # Don't use 'inspect' as it relies on an extension module, which break the
267
    # build thanks to 'warnings' being imported by setup.py.
268
    fxn_code = None
269
    if hasattr(showwarning, 'func_code'):
270
        fxn_code = showwarning.func_code
271
    elif hasattr(showwarning, '__func__'):
272
        fxn_code = showwarning.__func__.func_code
273
    if fxn_code:
274
        args = fxn_code.co_varnames[:fxn_code.co_argcount]
275
        CO_VARARGS = 0x4
276
        if 'line' not in args and not fxn_code.co_flags & CO_VARARGS:
277
            showwarning_msg = ("functions overriding warnings.showwarning() "
278
                                "must support the 'line' argument")
279
            if message == showwarning_msg:
280
                _show_warning(message, category, filename, lineno)
281
            else:
282
                warn(showwarning_msg, DeprecationWarning)
283
    # Print message and context
284
    showwarning(message, category, filename, lineno)
285
286
287
class WarningMessage(object):
288
289
    """Holds the result of a single showwarning() call."""
290
291
    _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
292
                        "line")
293
294
    def __init__(self, message, category, filename, lineno, file=None,
295
                    line=None):
296
        local_values = locals()
297
        for attr in self._WARNING_DETAILS:
298
            setattr(self, attr, local_values[attr])
299
        self._category_name = category.__name__ if category else None
300
301
    def __str__(self):
302
        return ("{message : %r, category : %r, filename : %r, lineno : %s, "
303
                    "line : %r}" % (self.message, self._category_name,
304
                                    self.filename, self.lineno, self.line))
305
306
307
class catch_warnings(object):
308
309
    """A context manager that copies and restores the warnings filter upon
310
    exiting the context.
311
312
    The 'record' argument specifies whether warnings should be captured by a
313
    custom implementation of warnings.showwarning() and be appended to a list
314
    returned by the context manager. Otherwise None is returned by the context
315
    manager. The objects appended to the list are arguments whose attributes
316
    mirror the arguments to showwarning().
317
318
    The 'module' argument is to specify an alternative module to the module
319
    named 'warnings' and imported under that name. This argument is only useful
320
    when testing the warnings module itself.
321
322
    """
323
1.4e-05 sec 324
    def __init__(self, record=False, module=None):
325
        """Specify whether to record warnings and if an alternative module
326
        should be used other than sys.modules['warnings'].
327
328
        For compatibility with Python 3.0, please consider all arguments to be
329
        keyword-only.
330
331
        """
3e-06 sec 332
        self._record = record
4e-06 sec 333
        self._module = sys.modules['warnings'] if module is None else module
7e-06 sec 334
        self._entered = False
335
336
    def __repr__(self):
337
        args = []
338
        if self._record:
339
            args.append("record=True")
340
        if self._module is not sys.modules['warnings']:
341
            args.append("module=%r" % self._module)
342
        name = type(self).__name__
343
        return "%s(%s)" % (name, ", ".join(args))
344
1.3e-05 sec 345
    def __enter__(self):
3e-06 sec 346
        if self._entered:
347
            raise RuntimeError("Cannot enter %r twice" % self)
3e-06 sec 348
        self._entered = True
2e-06 sec 349
        self._filters = self._module.filters
4e-06 sec 350
        self._module.filters = self._filters[:]
6e-06 sec 351
        self._showwarning = self._module.showwarning
4e-06 sec 352
        if self._record:
353
            log = []
354
            def showwarning(*args, **kwargs):
355
                log.append(WarningMessage(*args, **kwargs))
356
            self._module.showwarning = showwarning
357
            return log
358
        else:
3e-06 sec 359
            return None
360
1.2e-05 sec 361
    def __exit__(self, *exc_info):
2e-06 sec 362
        if not self._entered:
363
            raise RuntimeError("Cannot exit %r without entering first" % self)
3e-06 sec 364
        self._module.filters = self._filters
4e-06 sec 365
        self._module.showwarning = self._showwarning
366
367
368
# filters contains a sequence of filter 5-tuples
369
# The components of the 5-tuple are:
370
# - an action: error, ignore, always, default, module, or once
371
# - a compiled regex that must match the warning message
372
# - a class representing the warning category
373
# - a compiled regex that must match the module that is being warned
374
# - a line number for the line being warning, or 0 to mean any line
375
# If either if the compiled regexs are None, match anything.
376
_warnings_defaults = False
377
try:
378
    from _warnings import (filters, default_action, once_registry,
379
                            warn, warn_explicit)
380
    defaultaction = default_action
381
    onceregistry = once_registry
382
    _warnings_defaults = True
383
except ImportError:
384
    filters = []
385
    defaultaction = "default"
386
    onceregistry = {}
387
388
389
# Module initialization
390
_processoptions(sys.warnoptions)
391
if not _warnings_defaults:
392
    simplefilter("ignore", category=PendingDeprecationWarning, append=1)
393
    simplefilter("ignore", category=ImportWarning, append=1)
394
    bytes_warning = sys.flags.bytes_warning
395
    if bytes_warning > 1:
396
        bytes_action = "error"
397
    elif bytes_warning:
398
        bytes_action = "default"
399
    else:
400
        bytes_action = "ignore"
401
    simplefilter(bytes_action, category=BytesWarning, append=1)
402
del _warnings_defaults
/var/lib/python-support/python2.6/OpenSSL/__init__.py
     time      num code
1
#
2
# __init__.py
3
#
4
# Copyright (C) AB Strakt 2001, All rights reserved
5
#
6
# $Id: __init__.py,v 1.4 2004/07/22 12:01:25 martin Exp $
7
#
8
"""
9
pyOpenSSL - A simple wrapper around the OpenSSL library
0.000913 sec 10
"""
3e-06 sec 11
import rand, crypto, SSL, tsafe
2.2e-05 sec 12
from version import __version__
/var/lib/python-support/python2.6/OpenSSL/tsafe.py
     time      num code
0.003047 sec 1
from OpenSSL import SSL
8.2e-05 sec 2
_ssl = SSL
2e-06 sec 3
del SSL
4
4e-06 sec 5
import threading
5.4e-05 sec 6
_RLock = threading.RLock
4e-06 sec 7
del threading
8
1.3e-05 sec 9
class Connection:
2e-06 sec 10
    def __init__(self, *args):
11
        self._ssl_conn = apply(_ssl.Connection, args)
12
        self._lock = _RLock()
13
2e-06 sec 14
    for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read',
15
              'renegotiate', 'bind', 'listen', 'connect', 'accept',
16
              'setblocking', 'fileno', 'shutdown', 'close', 'get_cipher_list',
17
              'getpeername', 'getsockname', 'getsockopt', 'setsockopt',
18
              'makefile', 'get_app_data', 'set_app_data', 'state_string',
19
              'sock_shutdown', 'get_peer_certificate', 'want_read',
20
              'want_write', 'set_connect_state', 'set_accept_state',
0.000133 sec 21
              'connect_ex', 'sendall'):
22
        exec """def %s(self, *args):
23
            self._lock.acquire()
24
            try:
25
                return apply(self._ssl_conn.%s, args)
26
            finally:
6.9e-05 sec 27
                self._lock.release()\n""" % (f, f)
28
/var/lib/python-support/python2.6/OpenSSL/version.py
     time      num code
1
# Copyright (C) AB Strakt 2001-2004, All rights reserved
2
# Copyright (C) Jean-Paul Calderone 2008, All rights reserved
3
4
"""
5
pyOpenSSL - A simple wrapper around the OpenSSL library
0.000161 sec 6
"""
7
3e-06 sec 8
__version__ = '0.7'
/var/lib/python-support/python2.6/web/application.py
     time      num code
1
#!/usr/bin/python
2
"""
3
Web application
4
(from web.py)
5
"""
6
import webapi as web
7
import webapi, wsgi, utils
8
import debugerror
9
from utils import lstrips, safeunicode
10
import sys
11
12
import urllib
13
import traceback
14
import itertools
15
import os
16
import re
17
import types
18
19
try:
20
    import wsgiref.handlers
21
except ImportError:
22
    pass # don't break people with old Pythons
23
24
__all__ = [
25
    "application", "auto_application",
26
    "subdir_application", "subdomain_application", 
27
    "loadhook", "unloadhook",
28
    "autodelegate"
29
]
30
31
class application:
32
    """
33
    Application to delegate requests based on path.
34
    
35
        >>> urls = ("/hello", "hello")
36
        >>> app = application(urls, globals())
37
        >>> class hello:
38
        ...     def GET(self): return "hello"
39
        >>>
40
        >>> app.request("/hello").data
41
        'hello'
42
    """
43
    def __init__(self, mapping=(), fvars={}, autoreload=None):
44
        if autoreload is None:
45
            autoreload = web.config.get('debug', False)
46
        self.mapping = mapping
47
        self.fvars = fvars
48
        self.processors = []
49
50
        if autoreload:
51
            def main_module_name():
52
                mod = sys.modules['__main__']
53
                file = getattr(mod, '__file__', None) # make sure this works even from python interpreter
54
                return file and os.path.splitext(os.path.basename(file))[0]
55
56
            def modname(fvars):
57
                """find name of the module name from fvars."""
58
                file, name = fvars.get('__file__'), fvars.get('__name__')
59
                if file is None or name is None:
60
                    return None
61
62
                if name == '__main__':
63
                    # Since the __main__ module can't be reloaded, the module has 
64
                    # to be imported using its file name.                    
65
                    name = main_module_name()
66
                return name
67
                
68
            mapping_name = utils.dictfind(fvars, mapping)
69
            module_name = modname(fvars)
70
            
71
            def reload_mapping():
72
                """loadhook to reload mapping and fvars."""
73
                mod = __import__(module_name)
74
                mapping = getattr(mod, mapping_name, None)
75
                if mapping:
76
                    self.fvars = mod.__dict__
77
                    self.mapping = mapping
78
79
            self.add_processor(loadhook(Reloader()))
80
            if mapping_name and module_name:
81
                self.add_processor(loadhook(reload_mapping))
82
83
            # load __main__ module usings its filename, so that it can be reloaded.
84
            if main_module_name() and '__main__' in sys.argv:
85
                try:
86
                    __import__(main_module_name())
87
                except ImportError:
88
                    pass
89
90
    def add_mapping(self, pattern, classname):
91
        self.mapping += (pattern, classname)
92
        
93
    def add_processor(self, processor):
94
        """
95
        Adds a processor to the application. 
96
        
97
            >>> urls = ("/(.*)", "echo")
98
            >>> app = application(urls, globals())
99
            >>> class echo:
100
            ...     def GET(self, name): return name
101
            ...
102
            >>>
103
            >>> def hello(handler): return "hello, " +  handler()
104
            >>> app.add_processor(hello)
105
            >>> app.request("/web.py").data
106
            'hello, web.py'
107
        """
108
        self.processors.append(processor)
109
110
    def request(self, localpart='/', method='GET', data=None,
111
                host="0.0.0.0:8080", headers=None, https=False, **kw):
112
        """Makes request to this application for the specified path and method.
113
        Response will be a storage object with data, status and headers.
114
115
            >>> urls = ("/hello", "hello")
116
            >>> app = application(urls, globals())
117
            >>> class hello:
118
            ...     def GET(self): 
119
            ...         web.header('Content-Type', 'text/plain')
120
            ...         return "hello"
121
            ...
122
            >>> response = app.request("/hello")
123
            >>> response.data
124
            'hello'
125
            >>> response.status
126
            '200 OK'
127
            >>> response.headers['Content-Type']
128
            'text/plain'
129
130
        To use https, use https=True.
131
132
            >>> urls = ("/redirect", "redirect")
133
            >>> app = application(urls, globals())
134
            >>> class redirect:
135
            ...     def GET(self): raise web.seeother("/foo")
136
            ...
137
            >>> response = app.request("/redirect")
138
            >>> response.headers['Location']
139
            'http://0.0.0.0:8080/foo'
140
            >>> response = app.request("/redirect", https=True)
141
            >>> response.headers['Location']
142
            'https://0.0.0.0:8080/foo'
143
144
        The headers argument specifies HTTP headers as a mapping object
145
        such as a dict.
146
147
            >>> urls = ('/ua', 'uaprinter')
148
            >>> class uaprinter:
149
            ...     def GET(self):
150
            ...         return 'your user-agent is ' + web.ctx.env['HTTP_USER_AGENT']
151
            ... 
152
            >>> app = application(urls, globals())
153
            >>> app.request('/ua', headers = {
154
            ...      'User-Agent': 'a small jumping bean/1.0 (compatible)'
155
            ... }).data
156
            'your user-agent is a small jumping bean/1.0 (compatible)'
157
158
        """
159
        path, maybe_query = urllib.splitquery(localpart)
160
        query = maybe_query or ""
161
        
162
        if 'env' in kw:
163
            env = kw['env']
164
        else:
165
            env = {}
166
        env = dict(env, HTTP_HOST=host, REQUEST_METHOD=method, PATH_INFO=path, QUERY_STRING=query, HTTPS=str(https))
167
        headers = headers or {}
168
169
        for k, v in headers.items():
170
            env['HTTP_' + k.upper().replace('-', '_')] = v
171
172
        if 'HTTP_CONTENT_LENGTH' in env:
173
            env['CONTENT_LENGTH'] = env.pop('HTTP_CONTENT_LENGTH')
174
175
        if 'HTTP_CONTENT_TYPE' in env:
176
            env['CONTENT_TYPE'] = env.pop('HTTP_CONTENT_TYPE')
177
178
        if data:
179
            import StringIO
180
            if isinstance(data, dict):
181
                q = urllib.urlencode(data)
182
            else:
183
                q = data
184
            env['wsgi.input'] = StringIO.StringIO(q)
185
            if not env.get('CONTENT_TYPE', '').lower().startswith('multipart/') and 'CONTENT_LENGTH' not in env:
186
                env['CONTENT_LENGTH'] = len(q)
187
        response = web.storage()
188
        def start_response(status, headers):
189
            response.status = status
190
            response.headers = dict(headers)
191
            response.header_items = headers
192
        response.data = "".join(self.wsgifunc(cleanup_threadlocal=False)(env, start_response))
193
        return response
194
195
    def browser(self):
196
        import browser
197
        return browser.AppBrowser(self)
198
199
    def handle(self):
200
        fn, args = self._match(self.mapping, web.ctx.path)
201
        return self._delegate(fn, self.fvars, args)
202
        
203
    def handle_with_processors(self):
204
        def process(processors):
205
            try:
206
                web.ctx.app_stack.append(self)
207
                if processors:
208
                    p, processors = processors[0], processors[1:]
209
                    return p(lambda: process(processors))
210
                else:
211
                    return self.handle()
212
            except web.HTTPError:
213
                raise
214
            except:
215
                print >> web.debug, traceback.format_exc()
216
                raise self.internalerror()
217
                    
218
        try:
219
            # processors must be applied in the resvere order. (??)
220
            return process(self.processors)
221
        finally:
222
            web.ctx.app_stack = web.ctx.app_stack[:-1]
223
                        
1.2e-05 sec 224
    def wsgifunc(self, *middleware, **kw):
225
        """Returns a WSGI-compatible function for this application."""
2e-06 sec 226
        def peep(iterator):
227
            """Peeps into an iterator by doing an iteration
228
            and returns an equivalent iterator.
229
            """
230
            # wsgi requires the headers first
231
            # so we need to do an iteration
232
            # and save the result for later
233
            try:
234
                firstchunk = iterator.next()
235
            except StopIteration:
236
                firstchunk = ''
237
            
238
            return itertools.chain([firstchunk], iterator)    
239
                                
3e-06 sec 240
        def is_generator(x): return x and hasattr(x, 'next')
241
        
2e-06 sec 242
        def wsgi(env, start_resp):
243
            self.load(env)
244
245
            try:
246
                # allow uppercase methods only
247
                if web.ctx.method.upper() != web.ctx.method:
248
                    raise web.nomethod()
249
250
                result = self.handle_with_processors()
251
            except web.HTTPError, e:
252
                result = e.data
253
254
            if is_generator(result):
255
                result = peep(result)
256
            else:
257
                result = [utils.utf8(result)]
258
259
            status, headers = web.ctx.status, web.ctx.headers
260
            start_resp(status, headers)
261
262
            #@@@
263
            # Since the CherryPy Webserver uses thread pool, the thread-local state is never cleared.
264
            # This interferes with the other requests. 
265
            # clearing the thread-local storage to avoid that.
266
            # see utils.ThreadedDict for details
267
            import threading
268
            t = threading.currentThread()
269
            if kw.get('cleanup_threadlocal', True) and hasattr(t, '_d'):
270
                del t._d
271
        
272
            return result
273
3e-06 sec 274
        for m in middleware: 
275
            wsgi = m(wsgi)
276
4e-06 sec 277
        return wsgi
278
1.3e-05 sec 279
    def run(self, *middleware):
280
        """
281
        Starts handling requests. If called in a CGI or FastCGI context, it will follow
282
        that protocol. If called from the command line, it will start an HTTP
283
        server on the port named in the first command line argument, or, if there
284
        is no argument, on port 8080.
285
        
286
        `middleware` is a list of WSGI middleware which is applied to the resulting WSGI
287
        function.
288
        """
3e-06 sec 289
        return wsgi.runwsgi(self.wsgifunc(*middleware))
290
    
291
    def cgirun(self, *middleware):
292
        """
293
        Return a CGI handler. This is mostly useful with Google App Engine.
294
        There you can just do:
295
        
296
            main = app.cgirun()
297
        """
298
        wsgiapp = self.wsgifunc(*middleware)
299
300
        try:
301
            from google.appengine.ext.webapp.util import run_wsgi_app
302
            return run_wsgi_app(wsgiapp)
303
        except ImportError:
304
            # we're not running from within Google App Engine
305
            return wsgiref.handlers.CGIHandler().run(wsgiapp)
306
    
307
    def load(self, env):
308
        """Initializes ctx using env."""
309
        ctx = web.ctx
310
        ctx.clear()
311
        ctx.status = '200 OK'
312
        ctx.headers = []
313
        ctx.output = ''
314
        ctx.environ = ctx.env = env
315
        ctx.host = env.get('HTTP_HOST')
316
317
        if env.get('wsgi.url_scheme') in ['http', 'https']:
318
            ctx.protocol = env['wsgi.url_scheme']
319
        elif env.get('HTTPS', '').lower() in ['on', 'true', '1']:
320
            ctx.protocol = 'https'
321
        else:
322
            ctx.protocol = 'http'
323
        ctx.homedomain = ctx.protocol + '://' + env.get('HTTP_HOST', '[unknown]')
324
        ctx.homepath = os.environ.get('REAL_SCRIPT_NAME', env.get('SCRIPT_NAME', ''))
325
        ctx.home = ctx.homedomain + ctx.homepath
326
        #@@ home is changed when the request is handled to a sub-application.
327
        #@@ but the real home is required for doing absolute redirects.
328
        ctx.realhome = ctx.home
329
        ctx.ip = env.get('REMOTE_ADDR')
330
        ctx.method = env.get('REQUEST_METHOD')
331
        ctx.path = env.get('PATH_INFO')
332
        # http://trac.lighttpd.net/trac/ticket/406 requires:
333
        if env.get('SERVER_SOFTWARE', '').startswith('lighttpd/'):
334
            ctx.path = lstrips(env.get('REQUEST_URI').split('?')[0], ctx.homepath)
335
336
        if env.get('QUERY_STRING'):
337
            ctx.query = '?' + env.get('QUERY_STRING', '')
338
        else:
339
            ctx.query = ''
340
341
        ctx.fullpath = ctx.path + ctx.query
342
        
343
        for k, v in ctx.iteritems():
344
            if isinstance(v, str):
345
                ctx[k] = safeunicode(v)
346
347
        # status must always be str
348
        ctx.status = '200 OK'
349
        
350
        ctx.app_stack = []
351
352
    def _delegate(self, f, fvars, args=[]):
353
        def handle_class(cls):
354
            meth = web.ctx.method
355
            if meth == 'HEAD' and not hasattr(cls, meth):
356
                meth = 'GET'
357
            if not hasattr(cls, meth):
358
                raise web.nomethod(cls)
359
            tocall = getattr(cls(), meth)
360
            return tocall(*args)
361
            
362
        def is_class(o): return isinstance(o, (types.ClassType, type))
363
            
364
        if f is None:
365
            raise web.notfound()
366
        elif isinstance(f, application):
367
            return f.handle_with_processors()
368
        elif is_class(f):
369
            return handle_class(f)
370
        elif isinstance(f, basestring):
371
            if f.startswith('redirect '):
372
                url = f.split(' ', 1)[1]
373
                if web.ctx.method == "GET":
374
                    x = web.ctx.env.get('QUERY_STRING', '')
375
                    if x:
376
                        url += '?' + x
377
                raise web.redirect(url)
378
            elif '.' in f:
379
                x = f.split('.')
380
                mod, cls = '.'.join(x[:-1]), x[-1]
381
                mod = __import__(mod, globals(), locals(), [""])
382
                cls = getattr(mod, cls)
383
            else:
384
                cls = fvars[f]
385
            return handle_class(cls)
386
        elif hasattr(f, '__call__'):
387
            return f()
388
        else:
389
            return web.notfound()
390
391
    def _match(self, mapping, value):
392
        for pat, what in utils.group(mapping, 2):
393
            if isinstance(what, application):
394
                if value.startswith(pat):
395
                    f = lambda: self._delegate_sub_application(pat, what)
396
                    return f, None
397
                else:
398
                    continue
399
            elif isinstance(what, basestring):
400
                what, result = utils.re_subm('^' + pat + '$', what, value)
401
            else:
402
                result = utils.re_compile('^' + pat + '$').match(value)
403
                
404
            if result: # it's a match
405
                return what, [x and urllib.unquote(x) for x in result.groups()]
406
        return None, None
407
        
408
    def _delegate_sub_application(self, dir, app):
409
        """Deletes request to sub application `app` rooted at the directory `dir`.
410
        The home, homepath, path and fullpath values in web.ctx are updated to mimic request
411
        to the subapp and are restored after it is handled. 
412
        
413
        @@Any issues with when used with yield?
414
        """
415
        try:
416
            oldctx = web.storage(web.ctx)
417
            web.ctx.home += dir
418
            web.ctx.homepath += dir
419
            web.ctx.path = web.ctx.path[len(dir):]
420
            web.ctx.fullpath = web.ctx.fullpath[len(dir):]
421
            return app.handle_with_processors()
422
        finally:
423
            web.ctx.home = oldctx.home
424
            web.ctx.homepath = oldctx.homepath
425
            web.ctx.path = oldctx.path
426
            web.ctx.fullpath = oldctx.fullpath
427
            
428
    def get_parent_app(self):
429
        if self in web.ctx.app_stack:
430
            index = web.ctx.app_stack.index(self)
431
            if index > 0:
432
                return web.ctx.app_stack[index-1]
433
        
434
    def notfound(self):
435
        """Returns HTTPError with '404 not found' message"""
436
        parent = self.get_parent_app()
437
        if parent:
438
            return parent.notfound()
439
        else:
440
            return web._NotFound()
441
            
442
    def internalerror(self):
443
        """Returns HTTPError with '500 internal error' message"""
444
        parent = self.get_parent_app()
445
        if parent:
446
            return parent.internalerror()
447
        elif web.config.get('debug'):
448
            import debugerror
449
            return debugerror.debugerror()
450
        else:
451
            return web._InternalError()
452
453
class auto_application(application):
454
    """Application similar to `application` but urls are constructed 
455
    automatiacally using metaclass.
456
457
        >>> app = auto_application()
458
        >>> class hello(app.page):
459
        ...     def GET(self): return "hello, world"
460
        ...
461
        >>> class foo(app.page):
462
        ...     path = '/foo/.*'
463
        ...     def GET(self): return "foo"
464
        >>> app.request("/hello").data
465
        'hello, world'
466
        >>> app.request('/foo/bar').data
467
        'foo'
468
    """
469
    def __init__(self):
470
        application.__init__(self)
471
472
        class metapage(type):
473
            def __init__(klass, name, bases, attrs):
474
                type.__init__(klass, name, bases, attrs)
475
                path = attrs.get('path', '/' + name)
476
477
                # path can be specified as None to ignore that class
478
                # typically required to create a abstract base class.
479
                if path is not None:
480
                    self.add_mapping(path, klass)
481
482
        class page:
483
            path = None
484
            __metaclass__ = metapage
485
486
        self.page = page
487
488
# The application class already has the required functionality of subdir_application
489
subdir_application = application
490
                
491
class subdomain_application(application):
492
    """
493
    Application to delegate requests based on the host.
494
495
        >>> urls = ("/hello", "hello")
496
        >>> app = application(urls, globals())
497
        >>> class hello:
498
        ...     def GET(self): return "hello"
499
        >>>
500
        >>> mapping = ("hello.example.com", app)
501
        >>> app2 = subdomain_application(mapping)
502
        >>> app2.request("/hello", host="hello.example.com").data
503
        'hello'
504
        >>> response = app2.request("/hello", host="something.example.com")
505
        >>> response.status
506
        '404 Not Found'
507
        >>> response.data
508
        'not found'
509
    """
510
    def handle(self):
511
        host = web.ctx.host.split(':')[0] #strip port
512
        fn, args = self._match(self.mapping, host)
513
        return self._delegate(fn, self.fvars, args)
514
        
515
    def _match(self, mapping, value):
516
        for pat, what in utils.group(mapping, 2):
517
            if isinstance(what, basestring):
518
                what, result = utils.re_subm('^' + pat + '$', what, value)
519
            else:
520
                result = utils.re_compile('^' + pat + '$').match(value)
521
522
            if result: # it's a match
523
                return what, [x and urllib.unquote(x) for x in result.groups()]
524
        return None, None
525
        
526
def loadhook(h):
527
    """
528
    Converts a load hook into an application processor.
529
    
530
        >>> app = auto_application()
531
        >>> def f(): "something done before handling request"
532
        ...
533
        >>> app.add_processor(loadhook(f))
534
    """
535
    def processor(handler):
536
        h()
537
        return handler()
538
        
539
    return processor
540
    
541
def unloadhook(h):
542
    """
543
    Converts an unload hook into an application processor.
544
    
545
        >>> app = auto_application()
546
        >>> def f(): "something done after handling request"
547
        ...
548
        >>> app.add_processor(unloadhook(f))    
549
    """
550
    def processor(handler):
551
        try:
552
            return handler()
553
        finally:
554
            h()
555
            
556
    return processor
557
558
def autodelegate(prefix=''):
559
    """
560
    Returns a method that takes one argument and calls the method named prefix+arg,
561
    calling `notfound()` if there isn't one. Example:
562
563
        urls = ('/prefs/(.*)', 'prefs')
564
565
        class prefs:
566
            GET = autodelegate('GET_')
567
            def GET_password(self): pass
568
            def GET_privacy(self): pass
569
570
    `GET_password` would get called for `/prefs/password` while `GET_privacy` for 
571
    `GET_privacy` gets called for `/prefs/privacy`.
572
    
573
    If a user visits `/prefs/password/change` then `GET_password(self, '/change')`
574
    is called.
575
    """
576
    def internal(self, arg):
577
        if '/' in arg:
578
            first, rest = arg.split('/', 1)
579
            func = prefix + first
580
            args = ['/' + rest]
581
        else:
582
            func = prefix + arg
583
            args = []
584
        
585
        if hasattr(self, func):
586
            try:
587
                return getattr(self, func)(*args)
588
            except TypeError:
589
                return web.notfound()
590
        else:
591
            return web.notfound()
592
    return internal
593
594
class Reloader:
595
    """Checks to see if any loaded modules have changed on disk and, 
596
    if so, reloads them.
597
    """
598
    def __init__(self):
599
        self.mtimes = {}
600
601
    def __call__(self):
602
        for mod in sys.modules.values():
603
            self.check(mod)
604
            
605
    def check(self, mod):
606
        try: 
607
            mtime = os.stat(mod.__file__).st_mtime
608
        except (AttributeError, OSError, IOError):
609
            return
610
        if mod.__file__.endswith('.pyc') and os.path.exists(mod.__file__[:-1]):
611
            mtime = max(os.stat(mod.__file__[:-1]).st_mtime, mtime)
612
            
613
        if mod not in self.mtimes:
614
            self.mtimes[mod] = mtime
615
        elif self.mtimes[mod] < mtime:
616
            try: 
617
                reload(mod)
618
                self.mtimes[mod] = mtime
619
            except ImportError: 
620
                pass
621
622
if __name__ == "__main__":
623
    import doctest
624
    doctest.testmod()
/var/lib/python-support/python2.6/web/httpserver.py
     time      num code
1
__all__ = ["runsimple"]
2
3
import sys, os
4
import webapi as web
5
import net
6
import utils
7
8
def runbasic(func, server_address=("0.0.0.0", 8080)):
9
    """
10
    Runs a simple HTTP server hosting WSGI app `func`. The directory `static/` 
11
    is hosted statically.
12
13
    Based on [WsgiServer][ws] from [Colin Stewart][cs].
14
    
15
  [ws]: http://www.owlfish.com/software/wsgiutils/documentation/wsgi-server-api.html
16
  [cs]: http://www.owlfish.com/
17
    """
18
    # Copyright (c) 2004 Colin Stewart (http://www.owlfish.com/)
19
    # Modified somewhat for simplicity
20
    # Used under the modified BSD license:
21
    # http://www.xfree86.org/3.3.6/COPYRIGHT2.html#5
22
23
    import SimpleHTTPServer, SocketServer, BaseHTTPServer, urlparse
24
    import socket, errno
25
    import traceback
26
27
    class WSGIHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
28
        def run_wsgi_app(self):
29
            protocol, host, path, parameters, query, fragment = \
30
                urlparse.urlparse('http://dummyhost%s' % self.path)
31
32
            # we only use path, query
33
            env = {'wsgi.version': (1, 0)
34
                   ,'wsgi.url_scheme': 'http'
35
                   ,'wsgi.input': self.rfile
36
                   ,'wsgi.errors': sys.stderr
37
                   ,'wsgi.multithread': 1
38
                   ,'wsgi.multiprocess': 0
39
                   ,'wsgi.run_once': 0
40
                   ,'REQUEST_METHOD': self.command
41
                   ,'REQUEST_URI': self.path
42
                   ,'PATH_INFO': path
43
                   ,'QUERY_STRING': query
44
                   ,'CONTENT_TYPE': self.headers.get('Content-Type', '')
45
                   ,'CONTENT_LENGTH': self.headers.get('Content-Length', '')
46
                   ,'REMOTE_ADDR': self.client_address[0]
47
                   ,'SERVER_NAME': self.server.server_address[0]
48
                   ,'SERVER_PORT': str(self.server.server_address[1])
49
                   ,'SERVER_PROTOCOL': self.request_version
50
                   }
51
52
            for http_header, http_value in self.headers.items():
53
                env ['HTTP_%s' % http_header.replace('-', '_').upper()] = \
54
                    http_value
55
56
            # Setup the state
57
            self.wsgi_sent_headers = 0
58
            self.wsgi_headers = []
59
60
            try:
61
                # We have there environment, now invoke the application
62
                result = self.server.app(env, self.wsgi_start_response)
63
                try:
64
                    try:
65
                        for data in result:
66
                            if data: 
67
                                self.wsgi_write_data(data)
68
                    finally:
69
                        if hasattr(result, 'close'): 
70
                            result.close()
71
                except socket.error, socket_err:
72
                    # Catch common network errors and suppress them
73
                    if (socket_err.args[0] in \
74
                       (errno.ECONNABORTED, errno.EPIPE)): 
75
                        return
76
                except socket.timeout, socket_timeout: 
77
                    return
78
            except:
79
                print >> web.debug, traceback.format_exc(),
80
81
            if (not self.wsgi_sent_headers):
82
                # We must write out something!
83
                self.wsgi_write_data(" ")
84
            return
85
86
        do_POST = run_wsgi_app
87
        do_PUT = run_wsgi_app
88
        do_DELETE = run_wsgi_app
89
90
        def do_GET(self):
91
            if self.path.startswith('/static/'):
92
                SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
93
            else:
94
                self.run_wsgi_app()
95
96
        def wsgi_start_response(self, response_status, response_headers, 
97
                              exc_info=None):
98
            if (self.wsgi_sent_headers):
99
                raise Exception \
100
                      ("Headers already sent and start_response called again!")
101
            # Should really take a copy to avoid changes in the application....
102
            self.wsgi_headers = (response_status, response_headers)
103
            return self.wsgi_write_data
104
105
        def wsgi_write_data(self, data):
106
            if (not self.wsgi_sent_headers):
107
                status, headers = self.wsgi_headers
108
                # Need to send header prior to data
109
                status_code = status[:status.find(' ')]
110
                status_msg = status[status.find(' ') + 1:]
111
                self.send_response(int(status_code), status_msg)
112
                for header, value in headers:
113
                    self.send_header(header, value)
114
                self.end_headers()
115
                self.wsgi_sent_headers = 1
116
            # Send the data
117
            self.wfile.write(data)
118
119
    class WSGIServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer):
120
        def __init__(self, func, server_address):
121
            BaseHTTPServer.HTTPServer.__init__(self, 
122
                                               server_address, 
123
                                               WSGIHandler)
124
            self.app = func
125
            self.serverShuttingDown = 0
126
127
    print "http://%s:%d/" % server_address
128
    WSGIServer(func, server_address).serve_forever()
129
1.4e-05 sec 130
def runsimple(func, server_address=("0.0.0.0", 8080)):
131
    """
132
    Runs [CherryPy][cp] WSGI server hosting WSGI app `func`. 
133
    The directory `static/` is hosted statically.
134
135
    [cp]: http://www.cherrypy.org
136
    """
1e-06 sec 137
    from wsgiserver import CherryPyWSGIServer
7.1e-05 sec 138
    from SimpleHTTPServer import SimpleHTTPRequestHandler
3.1e-05 sec 139
    from BaseHTTPServer import BaseHTTPRequestHandler
140
7.9e-05 sec 141
    class StaticApp(SimpleHTTPRequestHandler):
3e-06 sec 142
        """WSGI application for serving static files."""
2e-06 sec 143
        def __init__(self, environ, start_response):
144
            self.headers = []
145
            self.environ = environ
146
            self.start_response = start_response
147
3e-06 sec 148
        def send_response(self, status, msg=""):
149
            self.status = str(status) + " " + msg
150
3e-06 sec 151
        def send_header(self, name, value):
152
            self.headers.append((name, value))
153
3e-06 sec 154
        def end_headers(self):
155
            pass
156
3e-06 sec 157
        def log_message(*a): pass
158
3e-06 sec 159
        def __iter__(self):
160
            environ = self.environ
161
162
            self.path = environ.get('PATH_INFO', '')
163
            self.client_address = environ.get('REMOTE_ADDR','-'), \
164
                                  environ.get('REMOTE_PORT','-')
165
            self.command = environ.get('REQUEST_METHOD', '-')
166
167
            from cStringIO import StringIO
168
            self.wfile = StringIO() # for capturing error
169
170
            f = self.send_head()
171
            self.start_response(self.status, self.headers)
172
173
            if f:
174
                block_size = 16 * 1024
175
                while True:
176
                    buf = f.read(block_size)
177
                    if not buf:
178
                        break
179
                    yield buf
180
                f.close()
181
            else:
182
                value = self.wfile.getvalue()
183
                yield value
184
                    
1.2e-05 sec 185
    class WSGIWrapper(BaseHTTPRequestHandler):
2e-06 sec 186
        """WSGI wrapper for logging the status and serving static files."""
1e-05 sec 187
        def __init__(self, app):
2e-06 sec 188
            self.app = app
3e-06 sec 189
            self.format = '%s - - [%s] "%s %s %s" - %s'
190
2e-06 sec 191
        def __call__(self, environ, start_response):
192
            def xstart_response(status, response_headers, *args):
193
                write = start_response(status, response_headers, *args)
194
                self.log(status, environ)
195
                return write
196
197
            path = environ.get('PATH_INFO', '')
198
            if path.startswith('/static/'):
199
                return StaticApp(environ, xstart_response)
200
            else:
201
                return self.app(environ, xstart_response)
202
3e-06 sec 203
        def log(self, status, environ):
204
            outfile = environ.get('wsgi.errors', web.debug)
205
            req = environ.get('PATH_INFO', '_')
206
            protocol = environ.get('ACTUAL_SERVER_PROTOCOL', '-')
207
            method = environ.get('REQUEST_METHOD', '-')
208
            host = "%s:%s" % (environ.get('REMOTE_ADDR','-'), 
209
                              environ.get('REMOTE_PORT','-'))
210
211
            #@@ It is really bad to extend from 
212
            #@@ BaseHTTPRequestHandler just for this method
213
            time = self.log_date_time_string()
214
215
            msg = self.format % (host, time, protocol, method, req, status)
216
            print >> outfile, utils.safestr(msg)
217
            
7e-06 sec 218
    func = WSGIWrapper(func)
3e-06 sec 219
    server = CherryPyWSGIServer(server_address, func, server_name="localhost")
220
4e-06 sec 221
    print "http://%s:%d/" % server_address
0.000101 sec 222
    try:
2e-06 sec 223
        server.start()
8e-06 sec 224
    except KeyboardInterrupt:
4e-06 sec 225
        server.stop()
/var/lib/python-support/python2.6/web/net.py
     time      num code
1
"""
2
Network Utilities
3
(from web.py)
4
"""
5
6
__all__ = [
7
  "validipaddr", "validipport", "validip", "validaddr", 
8
  "urlquote",
9
  "httpdate", "parsehttpdate", 
10
  "htmlquote", "htmlunquote", "websafe",
11
]
12
13
import urllib, time
14
try: import datetime
15
except ImportError: pass
16
17
def validipaddr(address):
18
    """
19
    Returns True if `address` is a valid IPv4 address.
20
    
21
        >>> validipaddr('192.168.1.1')
22
        True
23
        >>> validipaddr('192.168.1.800')
24
        False
25
        >>> validipaddr('192.168.1')
26
        False
27
    """
28
    try:
29
        octets = address.split('.')
30
        if len(octets) != 4:
31
            return False
32
        for x in octets:
33
            if not (0 <= int(x) <= 255):
34
                return False
35
    except ValueError:
36
        return False
37
    return True
38
39
def validipport(port):
40
    """
41
    Returns True if `port` is a valid IPv4 port.
42
    
43
        >>> validipport('9000')
44
        True
45
        >>> validipport('foo')
46
        False
47
        >>> validipport('1000000')
48
        False
49
    """
50
    try:
51
        if not (0 <= int(port) <= 65535):
52
            return False
53
    except ValueError:
54
        return False
55
    return True
56
8e-06 sec 57
def validip(ip, defaultaddr="0.0.0.0", defaultport=8080):
58
    """Returns `(ip_address, port)` from string `ip_addr_port`"""
3e-06 sec 59
    addr = defaultaddr
2e-06 sec 60
    port = defaultport
61
    
1e-06 sec 62
    ip = ip.split(":", 1)
8e-06 sec 63
    if len(ip) == 1:
3e-06 sec 64
        if not ip[0]:
2e-06 sec 65
            pass
66
        elif validipaddr(ip[0]):
67
            addr = ip[0]
68
        elif validipport(ip[0]):
69
            port = int(ip[0])
70
        else:
71
            raise ValueError, ':'.join(ip) + ' is not a valid IP address/port'
72
    elif len(ip) == 2:
73
        addr, port = ip
74
        if not validipaddr(addr) and validipport(port):
75
            raise ValueError, ':'.join(ip) + ' is not a valid IP address/port'
76
        port = int(port)
77
    else:
78
        raise ValueError, ':'.join(ip) + ' is not a valid IP address/port'
2e-06 sec 79
    return (addr, port)
80
81
def validaddr(string_):
82
    """
83
    Returns either (ip_address, port) or "/path/to/socket" from string_
84
    
85
        >>> validaddr('/path/to/socket')
86
        '/path/to/socket'
87
        >>> validaddr('8000')
88
        ('0.0.0.0', 8000)
89
        >>> validaddr('127.0.0.1')
90
        ('127.0.0.1', 8080)
91
        >>> validaddr('127.0.0.1:8000')
92
        ('127.0.0.1', 8000)
93
        >>> validaddr('fff')
94
        Traceback (most recent call last):
95
            ...
96
        ValueError: fff is not a valid IP address/port
97
    """
98
    if '/' in string_:
99
        return string_
100
    else:
101
        return validip(string_)
102
103
def urlquote(val):
104
    """
105
    Quotes a string for use in a URL.
106
    
107
        >>> urlquote('://?f=1&j=1')
108
        '%3A//%3Ff%3D1%26j%3D1'
109
        >>> urlquote(None)
110
        ''
111
        >>> urlquote(u'\u203d')
112
        '%E2%80%BD'
113
    """
114
    if val is None: return ''
115
    if not isinstance(val, unicode): val = str(val)
116
    else: val = val.encode('utf-8')
117
    return urllib.quote(val)
118
119
def httpdate(date_obj):
120
    """
121
    Formats a datetime object for use in HTTP headers.
122
    
123
        >>> import datetime
124
        >>> httpdate(datetime.datetime(1970, 1, 1, 1, 1, 1))
125
        'Thu, 01 Jan 1970 01:01:01 GMT'
126
    """
127
    return date_obj.strftime("%a, %d %b %Y %H:%M:%S GMT")
128
129
def parsehttpdate(string_):
130
    """
131
    Parses an HTTP date into a datetime object.
132
133
        >>> parsehttpdate('Thu, 01 Jan 1970 01:01:01 GMT')
134
        datetime.datetime(1970, 1, 1, 1, 1, 1)
135
    """
136
    try:
137
        t = time.strptime(string_, "%a, %d %b %Y %H:%M:%S %Z")
138
    except ValueError:
139
        return None
140
    return datetime.datetime(*t[:6])
141
142
def htmlquote(text):
143
    """
144
    Encodes `text` for raw use in HTML.
145
    
146
        >>> htmlquote("<'&\\">")
147
        '&lt;&#39;&amp;&quot;&gt;'
148
    """
149
    text = text.replace("&", "&amp;") # Must be done first!
150
    text = text.replace("<", "&lt;")
151
    text = text.replace(">", "&gt;")
152
    text = text.replace("'", "&#39;")
153
    text = text.replace('"', "&quot;")
154
    return text
155
156
def htmlunquote(text):
157
    """
158
    Decodes `text` that's HTML quoted.
159
160
        >>> htmlunquote('&lt;&#39;&amp;&quot;&gt;')
161
        '<\\'&">'
162
    """
163
    text = text.replace("&quot;", '"')
164
    text = text.replace("&#39;", "'")
165
    text = text.replace("&gt;", ">")
166
    text = text.replace("&lt;", "<")
167
    text = text.replace("&amp;", "&") # Must be done last!
168
    return text
169
170
def websafe(val):
171
    """
172
    Converts `val` so that it's safe for use in UTF-8 HTML.
173
    
174
        >>> websafe("<'&\\">")
175
        '&lt;&#39;&amp;&quot;&gt;'
176
        >>> websafe(None)
177
        ''
178
        >>> websafe(u'\u203d')
179
        '\\xe2\\x80\\xbd'
180
    """
181
    if val is None:
182
        return ''
183
    if isinstance(val, unicode):
184
        val = val.encode('utf-8')
185
    val = str(val)
186
    return htmlquote(val)
187
188
if __name__ == "__main__":
189
    import doctest
190
    doctest.testmod()
/var/lib/python-support/python2.6/web/utils.py
     time      num code
1
#!/usr/bin/env python
2
"""
3
General Utilities
4
(part of web.py)
5
"""
6
7
__all__ = [
8
  "Storage", "storage", "storify", 
9
  "iters", 
10
  "rstrips", "lstrips", "strips", 
11
  "safeunicode", "safestr", "utf8",
12
  "TimeoutError", "timelimit",
13
  "Memoize", "memoize",
14
  "re_compile", "re_subm",
15
  "group",
16
  "IterBetter", "iterbetter",
17
  "dictreverse", "dictfind", "dictfindall", "dictincr", "dictadd",
18
  "listget", "intget", "datestr",
19
  "numify", "denumify", "commify", "dateify",
20
  "nthstr",
21
  "CaptureStdout", "capturestdout", "Profile", "profile",
22
  "tryall",
23
  "ThreadedDict", "threadeddict",
24
  "autoassign",
25
  "to36",
26
  "safemarkdown",
27
  "sendmail"
28
]
29
30
import re, sys, time, threading
31
32
try:
33
    import subprocess
34
except ImportError: 
35
    subprocess = None
36
37
try: import datetime
38
except ImportError: pass
39
40
class Storage(dict):
41
    """
42
    A Storage object is like a dictionary except `obj.foo` can be used
43
    in addition to `obj['foo']`.
44
    
45
        >>> o = storage(a=1)
46
        >>> o.a
47
        1
48
        >>> o['a']
49
        1
50
        >>> o.a = 2
51
        >>> o['a']
52
        2
53
        >>> del o.a
54
        >>> o.a
55
        Traceback (most recent call last):
56
            ...
57
        AttributeError: 'a'
58
    
59
    """
60
    def __getattr__(self, key): 
61
        try:
62
            return self[key]
63
        except KeyError, k:
64
            raise AttributeError, k
65
    
66
    def __setattr__(self, key, value): 
67
        self[key] = value
68
    
69
    def __delattr__(self, key):
70
        try:
71
            del self[key]
72
        except KeyError, k:
73
            raise AttributeError, k
74
    
75
    def __repr__(self):     
76
        return '<Storage ' + dict.__repr__(self) + '>'
77
78
storage = Storage
79
80
def storify(mapping, *requireds, **defaults):
81
    """
82
    Creates a `storage` object from dictionary `mapping`, raising `KeyError` if
83
    d doesn't have all of the keys in `requireds` and using the default 
84
    values for keys found in `defaults`.
85
86
    For example, `storify({'a':1, 'c':3}, b=2, c=0)` will return the equivalent of
87
    `storage({'a':1, 'b':2, 'c':3})`.
88
    
89
    If a `storify` value is a list (e.g. multiple values in a form submission), 
90
    `storify` returns the last element of the list, unless the key appears in 
91
    `defaults` as a list. Thus:
92
    
93
        >>> storify({'a':[1, 2]}).a
94
        2
95
        >>> storify({'a':[1, 2]}, a=[]).a
96
        [1, 2]
97
        >>> storify({'a':1}, a=[]).a
98
        [1]
99
        >>> storify({}, a=[]).a
100
        []
101
    
102
    Similarly, if the value has a `value` attribute, `storify will return _its_
103
    value, unless the key appears in `defaults` as a dictionary.
104
    
105
        >>> storify({'a':storage(value=1)}).a
106
        1
107
        >>> storify({'a':storage(value=1)}, a={}).a
108
        <Storage {'value': 1}>
109
        >>> storify({}, a={}).a
110
        {}
111
        
112
    Optionally, keyword parameter `_unicode` can be passed to convert all values to unicode.
113
    
114
        >>> storify({'x': 'a'}, _unicode=True)
115
        <Storage {'x': u'a'}>
116
        >>> storify({'x': storage(value='a')}, x={}, _unicode=True)
117
        <Storage {'x': <Storage {'value': 'a'}>}>
118
        >>> storify({'x': storage(value='a')}, _unicode=True)
119
        <Storage {'x': u'a'}>
120
    """
121
    _unicode = defaults.pop('_unicode', False)
122
    def unicodify(s):
123
        if _unicode and isinstance(s, str): return safeunicode(s)
124
        else: return s
125
        
126
    def getvalue(x):
127
        if hasattr(x, 'value'):
128
            return unicodify(x.value)
129
        else:
130
            return unicodify(x)
131
    
132
    stor = Storage()
133
    for key in requireds + tuple(mapping.keys()):
134
        value = mapping[key]
135
        if isinstance(value, list):
136
            if isinstance(defaults.get(key), list):
137
                value = [getvalue(x) for x in value]
138
            else:
139
                value = value[-1]
140
        if not isinstance(defaults.get(key), dict):
141
            value = getvalue(value)
142
        if isinstance(defaults.get(key), list) and not isinstance(value, list):
143
            value = [value]
144
        setattr(stor, key, value)
145
146
    for (key, value) in defaults.iteritems():
147
        result = value
148
        if hasattr(stor, key): 
149
            result = stor[key]
150
        if value == () and not isinstance(result, tuple): 
151
            result = (result,)
152
        setattr(stor, key, result)
153
    
154
    return stor
155
156
iters = [list, tuple]
157
import __builtin__
158
if hasattr(__builtin__, 'set'):
159
    iters.append(set)
160
if hasattr(__builtin__, 'frozenset'):
161
    iters.append(set)
162
if sys.version_info < (2,6): # sets module deprecated in 2.6
163
    try:
164
        from sets import Set
165
        iters.append(Set)
166
    except ImportError: 
167
        pass
168
    
169
class _hack(tuple): pass
170
iters = _hack(iters)
171
iters.__doc__ = """
172
A list of iterable items (like lists, but not strings). Includes whichever
173
of lists, tuples, sets, and Sets are available in this version of Python.
174
"""
175
176
def _strips(direction, text, remove):
177
    if direction == 'l': 
178
        if text.startswith(remove): 
179
            return text[len(remove):]
180
    elif direction == 'r':
181
        if text.endswith(remove):   
182
            return text[:-len(remove)]
183
    else: 
184
        raise ValueError, "Direction needs to be r or l."
185
    return text
186
187
def rstrips(text, remove):
188
    """
189
    removes the string `remove` from the right of `text`
190
191
        >>> rstrips("foobar", "bar")
192
        'foo'
193
    
194
    """
195
    return _strips('r', text, remove)
196
197
def lstrips(text, remove):
198
    """
199
    removes the string `remove` from the left of `text`
200
    
201
        >>> lstrips("foobar", "foo")
202
        'bar'
203
    
204
    """
205
    return _strips('l', text, remove)
206
207
def strips(text, remove):
208
    """
209
    removes the string `remove` from the both sides of `text`
210
211
        >>> strips("foobarfoo", "foo")
212
        'bar'
213
    
214
    """
215
    return rstrips(lstrips(text, remove), remove)
216
217
def safeunicode(obj, encoding='utf-8'):
218
    r"""
219
    Converts any given object to unicode string.
220
    
221
        >>> safeunicode('hello')
222
        u'hello'
223
        >>> safeunicode(2)
224
        u'2'
225
        >>> safeunicode('\xe1\x88\xb4')
226
        u'\u1234'
227
    """
228
    if isinstance(obj, unicode):
229
        return obj
230
    elif isinstance(obj, str):
231
        return obj.decode(encoding)
232
    else:
233
        if hasattr(obj, '__unicode__'):
234
            return unicode(obj)
235
        else:
236
            return str(obj).decode(encoding)
237
    
238
def safestr(obj, encoding='utf-8'):
239
    r"""
240
    Converts any given object to utf-8 encoded string. 
241
    
242
        >>> safestr('hello')
243
        'hello'
244
        >>> safestr(u'\u1234')
245
        '\xe1\x88\xb4'
246
        >>> safestr(2)
247
        '2'
248
    """
249
    if isinstance(obj, unicode):
250
        return obj.encode('utf-8')
251
    elif isinstance(obj, str):
252
        return obj
253
    else:
254
        return str(obj)
255
256
# for backward-compatibility
257
utf8 = safestr
258
    
259
class TimeoutError(Exception): pass
260
def timelimit(timeout):
261
    """
262
    A decorator to limit a function to `timeout` seconds, raising `TimeoutError`
263
    if it takes longer.
264
    
265
        >>> import time
266
        >>> def meaningoflife():
267
        ...     time.sleep(.2)
268
        ...     return 42
269
        >>> 
270
        >>> timelimit(.1)(meaningoflife)()
271
        Traceback (most recent call last):
272
            ...
273
        TimeoutError: took too long
274
        >>> timelimit(1)(meaningoflife)()
275
        42
276
277
    _Caveat:_ The function isn't stopped after `timeout` seconds but continues 
278
    executing in a separate thread. (There seems to be no way to kill a thread.)
279
280
    inspired by <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473878>
281
    """
282
    def _1(function):
283
        def _2(*args, **kw):
284
            class Dispatch(threading.Thread):
285
                def __init__(self):
286
                    threading.Thread.__init__(self)
287
                    self.result = None
288
                    self.error = None
289
290
                    self.setDaemon(True)
291
                    self.start()
292
293
                def run(self):
294
                    try:
295
                        self.result = function(*args, **kw)
296
                    except:
297
                        self.error = sys.exc_info()
298
299
            c = Dispatch()
300
            c.join(timeout)
301
            if c.isAlive():
302
                raise TimeoutError, 'took too long'
303
            if c.error:
304
                raise c.error[0], c.error[1]
305
            return c.result
306
        return _2
307
    return _1
308
309
class Memoize:
310
    """
311
    'Memoizes' a function, caching its return values for each input.
312
    
313
        >>> import time
314
        >>> def meaningoflife():
315
        ...     time.sleep(.2)
316
        ...     return 42
317
        >>> fastlife = memoize(meaningoflife)
318
        >>> meaningoflife()
319
        42
320
        >>> timelimit(.1)(meaningoflife)()
321
        Traceback (most recent call last):
322
            ...
323
        TimeoutError: took too long
324
        >>> fastlife()
325
        42
326
        >>> timelimit(.1)(fastlife)()
327
        42
328
    
329
    """
330
    def __init__(self, func): 
331
        self.func = func
332
        self.cache = {}
333
    def __call__(self, *args, **keywords):
334
        key = (args, tuple(keywords.items()))
335
        if key not in self.cache: 
336
            self.cache[key] = self.func(*args, **keywords)
337
        return self.cache[key]
338
339
memoize = Memoize
340
341
re_compile = memoize(re.compile) #@@ threadsafe?
342
re_compile.__doc__ = """
343
A memoized version of re.compile.
344
"""
345
346
class _re_subm_proxy:
347
    def __init__(self): 
348
        self.match = None
349
    def __call__(self, match): 
350
        self.match = match
351
        return ''
352
353
def re_subm(pat, repl, string):
354
    """
355
    Like re.sub, but returns the replacement _and_ the match object.
356
    
357
        >>> t, m = re_subm('g(oo+)fball', r'f\\1lish', 'goooooofball')
358
        >>> t
359
        'foooooolish'
360
        >>> m.groups()
361
        ('oooooo',)
362
    """
363
    compiled_pat = re_compile(pat)
364
    proxy = _re_subm_proxy()
365
    compiled_pat.sub(proxy.__call__, string)
366
    return compiled_pat.sub(repl, string), proxy.match
367
368
def group(seq, size): 
369
    """
370
    Returns an iterator over a series of lists of length size from iterable.
371
372
        >>> list(group([1,2,3,4], 2))
373
        [[1, 2], [3, 4]]
374
    """
375
    if not hasattr(seq, 'next'):  
376
        seq = iter(seq)
377
    while True: 
378
        yield [seq.next() for i in xrange(size)]
379
380
class IterBetter:
381
    """
382
    Returns an object that can be used as an iterator 
383
    but can also be used via __getitem__ (although it 
384
    cannot go backwards -- that is, you cannot request 
385
    `iterbetter[0]` after requesting `iterbetter[1]`).
386
    
387
        >>> import itertools
388
        >>> c = iterbetter(itertools.count())
389
        >>> c[1]
390
        1
391
        >>> c[5]
392
        5
393
        >>> c[3]
394
        Traceback (most recent call last):
395
            ...
396
        IndexError: already passed 3
397
    """
398
    def __init__(self, iterator): 
399
        self.i, self.c = iterator, 0
400
    def __iter__(self): 
401
        while 1:    
402
            yield self.i.next()
403
            self.c += 1
404
    def __getitem__(self, i):
405
        #todo: slices
406
        if i < self.c: 
407
            raise IndexError, "already passed "+str(i)
408
        try:
409
            while i > self.c: 
410
                self.i.next()
411
                self.c += 1
412
            # now self.c == i
413
            self.c += 1
414
            return self.i.next()
415
        except StopIteration: 
416
            raise IndexError, str(i)
417
iterbetter = IterBetter
418
419
def dictreverse(mapping):
420
    """
421
    Returns a new dictionary with keys and values swapped.
422
    
423
        >>> dictreverse({1: 2, 3: 4})
424
        {2: 1, 4: 3}
425
    """
426
    return dict([(value, key) for (key, value) in mapping.iteritems()])
427
428
def dictfind(dictionary, element):
429
    """
430
    Returns a key whose value in `dictionary` is `element` 
431
    or, if none exists, None.
432
    
433
        >>> d = {1:2, 3:4}
434
        >>> dictfind(d, 4)
435
        3
436
        >>> dictfind(d, 5)
437
    """
438
    for (key, value) in dictionary.iteritems():
439
        if element is value: 
440
            return key
441
442
def dictfindall(dictionary, element):
443
    """
444
    Returns the keys whose values in `dictionary` are `element`
445
    or, if none exists, [].
446
    
447
        >>> d = {1:4, 3:4}
448
        >>> dictfindall(d, 4)
449
        [1, 3]
450
        >>> dictfindall(d, 5)
451
        []
452
    """
453
    res = []
454
    for (key, value) in dictionary.iteritems():
455
        if element is value:
456
            res.append(key)
457
    return res
458
459
def dictincr(dictionary, element):
460
    """
461
    Increments `element` in `dictionary`, 
462
    setting it to one if it doesn't exist.
463
    
464
        >>> d = {1:2, 3:4}
465
        >>> dictincr(d, 1)
466
        3
467
        >>> d[1]
468
        3
469
        >>> dictincr(d, 5)
470
        1
471
        >>> d[5]
472
        1
473
    """
474
    dictionary.setdefault(element, 0)
475
    dictionary[element] += 1
476
    return dictionary[element]
477
478
def dictadd(*dicts):
479
    """
480
    Returns a dictionary consisting of the keys in the argument dictionaries.
481
    If they share a key, the value from the last argument is used.
482
    
483
        >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1})
484
        {1: 0, 2: 1, 3: 1}
485
    """
486
    result = {}
487
    for dct in dicts:
488
        result.update(dct)
489
    return result
490
9e-06 sec 491
def listget(lst, ind, default=None):
492
    """
493
    Returns `lst[ind]` if it exists, `default` otherwise.
494
    
495
        >>> listget(['a'], 0)
496
        'a'
497
        >>> listget(['a'], 1)
498
        >>> listget(['a'], 1, 'b')
499
        'b'
500
    """
3e-06 sec 501
    if len(lst)-1 < ind: 
5e-06 sec 502
        return default
503
    return lst[ind]
504
505
def intget(integer, default=None):
506
    """
507
    Returns `integer` as an int or `default` if it can't.
508
    
509
        >>> intget('3')
510
        3
511
        >>> intget('3a')
512
        >>> intget('3a', 0)
513
        0
514
    """
515
    try:
516
        return int(integer)
517
    except (TypeError, ValueError):
518
        return default
519
520
def datestr(then, now=None):
521
    """
522
    Converts a (UTC) datetime object to a nice string representation.
523
    
524
        >>> from datetime import datetime, timedelta
525
        >>> d = datetime(1970, 5, 1)
526
        >>> datestr(d, now=d)
527
        '0 microseconds ago'
528
        >>> for t, v in {
529
        ...   timedelta(microseconds=1): '1 microsecond ago',
530
        ...   timedelta(microseconds=2): '2 microseconds ago',
531
        ...   -timedelta(microseconds=1): '1 microsecond from now',
532
        ...   -timedelta(microseconds=2): '2 microseconds from now',
533
        ...   timedelta(microseconds=2000): '2 milliseconds ago',
534
        ...   timedelta(seconds=2): '2 seconds ago',
535
        ...   timedelta(seconds=2*60): '2 minutes ago',
536
        ...   timedelta(seconds=2*60*60): '2 hours ago',
537
        ...   timedelta(days=2): '2 days ago',
538
        ... }.iteritems():
539
        ...     assert datestr(d, now=d+t) == v
540
        >>> datestr(datetime(1970, 1, 1), now=d)
541
        'January  1'
542
        >>> datestr(datetime(1969, 1, 1), now=d)
543
        'January  1, 1969'
544
        >>> datestr(datetime(1970, 6, 1), now=d)
545
        'June  1, 1970'
546
    """
547
    def agohence(n, what, divisor=None):
548
        if divisor: n = n // divisor
549
550
        out = str(abs(n)) + ' ' + what       # '2 day'
551
        if abs(n) != 1: out += 's'           # '2 days'
552
        out += ' '                           # '2 days '
553
        if n < 0:
554
            out += 'from now'
555
        else:
556
            out += 'ago'
557
        return out                           # '2 days ago'
558
559
    oneday = 24 * 60 * 60
560
561
    if not now: now = datetime.datetime.utcnow()
562
    if type(now).__name__ == "DateTime":
563
        now = datetime.datetime.fromtimestamp(now)
564
    if type(then).__name__ == "DateTime":
565
        then = datetime.datetime.fromtimestamp(then)
566
    delta = now - then
567
    deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06)
568
    deltadays = abs(deltaseconds) // oneday
569
    if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor
570
571
    if deltadays:
572
        if abs(deltadays) < 4:
573
            return agohence(deltadays, 'day')
574
575
        out = then.strftime('%B %e') # e.g. 'June 13'
576
        if then.year != now.year or deltadays < 0:
577
            out += ', %s' % then.year
578
        return out
579
580
    if int(deltaseconds):
581
        if abs(deltaseconds) > (60 * 60):
582
            return agohence(deltaseconds, 'hour', 60 * 60)
583
        elif abs(deltaseconds) > 60:
584
            return agohence(deltaseconds, 'minute', 60)
585
        else:
586
            return agohence(deltaseconds, 'second')
587
588
    deltamicroseconds = delta.microseconds
589
    if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity
590
    if abs(deltamicroseconds) > 1000:
591
        return agohence(deltamicroseconds, 'millisecond', 1000)
592
593
    return agohence(deltamicroseconds, 'microsecond')
594
595
def numify(string):
596
    """
597
    Removes all non-digit characters from `string`.
598
    
599
        >>> numify('800-555-1212')
600
        '8005551212'
601
        >>> numify('800.555.1212')
602
        '8005551212'
603
    
604
    """
605
    return ''.join([c for c in str(string) if c.isdigit()])
606
607
def denumify(string, pattern):
608
    """
609
    Formats `string` according to `pattern`, where the letter X gets replaced
610
    by characters from `string`.
611
    
612
        >>> denumify("8005551212", "(XXX) XXX-XXXX")
613
        '(800) 555-1212'
614
    
615
    """
616
    out = []
617
    for c in pattern:
618
        if c == "X":
619
            out.append(string[0])
620
            string = string[1:]
621
        else:
622
            out.append(c)
623
    return ''.join(out)
624
625
def commify(n):
626
    """
627
    Add commas to an integer `n`.
628
629
        >>> commify(1)
630
        '1'
631
        >>> commify(123)
632
        '123'
633
        >>> commify(1234)
634
        '1,234'
635
        >>> commify(1234567890)
636
        '1,234,567,890'
637
        >>> commify(None)
638
        >>>
639
    
640
    """
641
    if n is None: return None
642
    r = []
643
    for i, c in enumerate(reversed(str(n))):
644
        if i and (not (i % 3)):
645
            r.insert(0, ',')
646
        r.insert(0, c)
647
    return ''.join(r)
648
649
def dateify(datestring):
650
    """
651
    Formats a numified `datestring` properly.
652
    """
653
    return denumify(datestring, "XXXX-XX-XX XX:XX:XX")
654
655
656
def nthstr(n):
657
    """
658
    Formats an ordinal.
659
    Doesn't handle negative numbers.
660
661
        >>> nthstr(1)
662
        '1st'
663
        >>> nthstr(0)
664
        '0th'
665
        >>> [nthstr(x) for x in [2, 3, 4, 5, 10, 11, 12, 13, 14, 15]]
666
        ['2nd', '3rd', '4th', '5th', '10th', '11th', '12th', '13th', '14th', '15th']
667
        >>> [nthstr(x) for x in [91, 92, 93, 94, 99, 100, 101, 102]]
668
        ['91st', '92nd', '93rd', '94th', '99th', '100th', '101st', '102nd']
669
        >>> [nthstr(x) for x in [111, 112, 113, 114, 115]]
670
        ['111th', '112th', '113th', '114th', '115th']
671
672
    """
673
    
674
    assert n >= 0
675
    if n % 100 in [11, 12, 13]: return '%sth' % n
676
    return {1: '%sst', 2: '%snd', 3: '%srd'}.get(n % 10, '%sth') % n
677
678
def cond(predicate, consequence, alternative=None):
679
    """
680
    Function replacement for if-else to use in expressions.
681
        
682
        >>> x = 2
683
        >>> cond(x % 2 == 0, "even", "odd")
684
        'even'
685
        >>> cond(x % 2 == 0, "even", "odd") + '_row'
686
        'even_row'
687
    """
688
    if predicate:
689
        return consequence
690
    else:
691
        return alternative
692
693
class CaptureStdout:
694
    """
695
    Captures everything `func` prints to stdout and returns it instead.
696
    
697
        >>> def idiot():
698
        ...     print "foo"
699
        >>> capturestdout(idiot)()
700
        'foo\\n'
701
    
702
    **WARNING:** Not threadsafe!
703
    """
704
    def __init__(self, func): 
705
        self.func = func
706
    def __call__(self, *args, **keywords):
707
        from cStringIO import StringIO
708
        # Not threadsafe!
709
        out = StringIO()
710
        oldstdout = sys.stdout
711
        sys.stdout = out
712
        try: 
713
            self.func(*args, **keywords)
714
        finally: 
715
            sys.stdout = oldstdout
716
        return out.getvalue()
717
718
capturestdout = CaptureStdout
719
720
class Profile:
721
    """
722
    Profiles `func` and returns a tuple containing its output
723
    and a string with human-readable profiling information.
724
        
725
        >>> import time
726
        >>> out, inf = profile(time.sleep)(.001)
727
        >>> out
728
        >>> inf[:10].strip()
729
        'took 0.0'
730
    """
731
    def __init__(self, func): 
732
        self.func = func
733
    def __call__(self, *args): ##, **kw):   kw unused
734
        import hotshot, hotshot.stats, tempfile ##, time already imported
735
        temp = tempfile.NamedTemporaryFile()
736
        prof = hotshot.Profile(temp.name)
737
738
        stime = time.time()
739
        result = prof.runcall(self.func, *args)
740
        stime = time.time() - stime
741
        prof.close()
742
743
        import cStringIO
744
        out = cStringIO.StringIO()
745
        stats = hotshot.stats.load(temp.name)
746
        stats.stream = out
747
        stats.strip_dirs()
748
        stats.sort_stats('time', 'calls')
749
        stats.print_stats(40)
750
        stats.print_callers()
751
752
        x =  '\n\ntook '+ str(stime) + ' seconds\n'
753
        x += out.getvalue()
754
755
        return result, x
756
757
profile = Profile
758
759
760
import traceback
761
# hack for compatibility with Python 2.3:
762
if not hasattr(traceback, 'format_exc'):
763
    from cStringIO import StringIO
764
    def format_exc(limit=None):
765
        strbuf = StringIO()
766
        traceback.print_exc(limit, strbuf)
767
        return strbuf.getvalue()
768
    traceback.format_exc = format_exc
769
770
def tryall(context, prefix=None):
771
    """
772
    Tries a series of functions and prints their results. 
773
    `context` is a dictionary mapping names to values; 
774
    the value will only be tried if it's callable.
775
    
776
        >>> tryall(dict(j=lambda: True))
777
        j: True
778
        ----------------------------------------
779
        results:
780
           True: 1
781
782
    For example, you might have a file `test/stuff.py` 
783
    with a series of functions testing various things in it. 
784
    At the bottom, have a line:
785
786
        if __name__ == "__main__": tryall(globals())
787
788
    Then you can run `python test/stuff.py` and get the results of 
789
    all the tests.
790
    """
791
    context = context.copy() # vars() would update
792
    results = {}
793
    for (key, value) in context.iteritems():
794
        if not hasattr(value, '__call__'): 
795
            continue
796
        if prefix and not key.startswith(prefix): 
797
            continue
798
        print key + ':',
799
        try:
800
            r = value()
801
            dictincr(results, r)
802
            print r
803
        except:
804
            print 'ERROR'
805
            dictincr(results, 'ERROR')
806
            print '   ' + '\n   '.join(traceback.format_exc().split('\n'))
807
        
808
    print '-'*40
809
    print 'results:'
810
    for (key, value) in results.iteritems():
811
        print ' '*2, str(key)+':', value
812
        
813
class ThreadedDict:
814
    """
815
    Thread local storage.
816
    
817
        >>> d = ThreadedDict()
818
        >>> d.x = 1
819
        >>> d.x
820
        1
821
        >>> import threading
822
        >>> def f(): d.x = 2
823
        >>> t = threading.Thread(target=f)
824
        >>> t.start()
825
        >>> t.join()
826
        >>> d.x
827
        1
828
    """
829
    def __getattr__(self, key):
830
        return getattr(self._getd(), key)
831
832
    def __setattr__(self, key, value):
833
        return setattr(self._getd(), key, value)
834
835
    def __delattr__(self, key):
836
        return delattr(self._getd(), key)
837
838
    def __hash__(self): 
839
        return id(self)
840
841
    def _getd(self):
842
        t = threading.currentThread()
843
        if not hasattr(t, '_d'):
844
            # using __dict__ of thread as thread local storage
845
            t._d = {}
846
847
        # there could be multiple instances of ThreadedDict.
848
        # use self as key
849
        if self not in t._d:
850
            t._d[self] = storage()
851
        return t._d[self]
852
853
threadeddict = ThreadedDict
854
855
def autoassign(self, locals):
856
    """
857
    Automatically assigns local variables to `self`.
858
    
859
        >>> self = storage()
860
        >>> autoassign(self, dict(a=1, b=2))
861
        >>> self
862
        <Storage {'a': 1, 'b': 2}>
863
    
864
    Generally used in `__init__` methods, as in:
865
866
        def __init__(self, foo, bar, baz=1): autoassign(self, locals())
867
    """
868
    for (key, value) in locals.iteritems():
869
        if key == 'self': 
870
            continue
871
        setattr(self, key, value)
872
873
def to36(q):
874
    """
875
    Converts an integer to base 36 (a useful scheme for human-sayable IDs).
876
    
877
        >>> to36(35)
878
        'z'
879
        >>> to36(119292)
880
        '2k1o'
881
        >>> int(to36(939387374), 36)
882
        939387374
883
        >>> to36(0)
884
        '0'
885
        >>> to36(-393)
886
        Traceback (most recent call last):
887
            ... 
888
        ValueError: must supply a positive integer
889
    
890
    """
891
    if q < 0: raise ValueError, "must supply a positive integer"
892
    letters = "0123456789abcdefghijklmnopqrstuvwxyz"
893
    converted = []
894
    while q != 0:
895
        q, r = divmod(q, 36)
896
        converted.insert(0, letters[r])
897
    return "".join(converted) or '0'
898
899
900
r_url = re_compile('(?<!\()(http://(\S+))')
901
def safemarkdown(text):
902
    """
903
    Converts text to HTML following the rules of Markdown, but blocking any
904
    outside HTML input, so that only the things supported by Markdown
905
    can be used. Also converts raw URLs to links.
906
907
    (requires [markdown.py](http://webpy.org/markdown.py))
908
    """
909
    from markdown import markdown
910
    if text:
911
        text = text.replace('<', '&lt;')
912
        # TODO: automatically get page title?
913
        text = r_url.sub(r'<\1>', text)
914
        text = markdown(text)
915
        return text
916
917
def sendmail(from_address, to_address, subject, message, headers=None, **kw):
918
    """
919
    Sends the email message `message` with mail and envelope headers
920
    for from `from_address_` to `to_address` with `subject`. 
921
    Additional email headers can be specified with the dictionary 
922
    `headers.
923
924
    If `web.config.smtp_server` is set, it will send the message
925
    to that SMTP server. Otherwise it will look for 
926
    `/usr/sbin/sendmail`, the typical location for the sendmail-style
927
    binary. To use sendmail from a different path, set `web.config.sendmail_path`.
928
    """
929
    try:
930
        import webapi
931
    except ImportError:
932
        webapi = Storage(config=Storage())
933
    
934
    if headers is None: headers = {}
935
    
936
    cc = kw.get('cc', [])
937
    bcc = kw.get('bcc', [])
938
    
939
    def listify(x):
940
        if not isinstance(x, list):
941
            return [safestr(x)]
942
        else:
943
            return [safestr(a) for a in x]
944
945
    from_address = safestr(from_address)
946
947
    to_address = listify(to_address)
948
    cc = listify(cc)
949
    bcc = listify(bcc)
950
951
    recipients = to_address + cc + bcc
952
    
953
    headers = dictadd({
954
      'MIME-Version': '1.0',
955
      'Content-Type': 'text/plain; charset=UTF-8',
956
      'Content-Disposition': 'inline',
957
      'From': from_address,
958
      'To': ", ".join(to_address),
959
      'Subject': subject
960
    }, headers)
961
962
    if cc:
963
        headers['Cc'] = ", ".join(cc)
964
    
965
    import email.Utils
966
    from_address = email.Utils.parseaddr(from_address)[1]
967
    recipients = [email.Utils.parseaddr(r)[1] for r in recipients]
968
    message = ('\n'.join([safestr('%s: %s' % x) for x in headers.iteritems()])
969
      + "\n\n" +  safestr(message))
970
971
    if webapi.config.get('smtp_server'):
972
        server = webapi.config.get('smtp_server')
973
        port = webapi.config.get('smtp_port', 0)
974
        username = webapi.config.get('smtp_username') 
975
        password = webapi.config.get('smtp_password')
976
        debug_level = webapi.config.get('smtp_debuglevel', None)
977
        starttls = webapi.config.get('smtp_starttls', False)
978
979
        import smtplib
980
        smtpserver = smtplib.SMTP(server, port)
981
982
        if debug_level:
983
            smtpserver.set_debuglevel(debug_level)
984
985
        if starttls:
986
            smtpserver.ehlo()
987
            smtpserver.starttls()
988
            smtpserver.ehlo()
989
990
        if username and password:
991
            smtpserver.login(username, password)
992
993
        smtpserver.sendmail(from_address, recipients, message)
994
        smtpserver.quit()
995
    else:
996
        sendmail = webapi.config.get('sendmail_path', '/usr/sbin/sendmail')
997
        
998
        assert not from_address.startswith('-'), 'security'
999
        for r in recipients:
1000
            assert not r.startswith('-'), 'security'
1001
                
1002
1003
        if subprocess:
1004
            p = subprocess.Popen(['/usr/sbin/sendmail', '-f', from_address] + recipients, stdin=subprocess.PIPE)
1005
            p.stdin.write(message)
1006
            p.stdin.close()
1007
            p.wait()
1008
        else:
1009
            i, o = os.popen2(["/usr/lib/sendmail", '-f', from_address] + recipients)
1010
            i.write(message)
1011
            i.close()
1012
            o.close()
1013
            del i, o
1014
1015
if __name__ == "__main__":
1016
    import doctest
1017
    doctest.testmod()
/var/lib/python-support/python2.6/web/wsgi.py
     time      num code
1
"""
2
WSGI Utilities
3
(from web.py)
4
"""
5
6
import os, sys
7
8
import http
9
import webapi as web
10
from utils import listget
11
from net import validaddr, validip
12
import httpserver
13
    
14
def runfcgi(func, addr=('localhost', 8000)):
15
    """Runs a WSGI function as a FastCGI server."""
16
    import flup.server.fcgi as flups
17
    return flups.WSGIServer(func, multiplexed=True, bindAddress=addr).run()
18
19
def runscgi(func, addr=('localhost', 4000)):
20
    """Runs a WSGI function as an SCGI server."""
21
    import flup.server.scgi as flups
22
    return flups.WSGIServer(func, bindAddress=addr).run()
23
1.9e-05 sec 24
def runwsgi(func):
25
    """
26
    Runs a WSGI-compatible `func` using FCGI, SCGI, or a simple web server,
27
    as appropriate based on context and `sys.argv`.
28
    """
29
    
2e-06 sec 30
    if os.environ.has_key('SERVER_SOFTWARE'): # cgi
31
        os.environ['FCGI_FORCE_CGI'] = 'Y'
32
2e-06 sec 33
    if (os.environ.has_key('PHP_FCGI_CHILDREN') #lighttpd fastcgi
2e-06 sec 34
      or os.environ.has_key('SERVER_SOFTWARE')):
35
        return runfcgi(func, None)
36
    
2e-06 sec 37
    if 'fcgi' in sys.argv or 'fastcgi' in sys.argv:
38
        args = sys.argv[1:]
39
        if 'fastcgi' in args: args.remove('fastcgi')
40
        elif 'fcgi' in args: args.remove('fcgi')
41
        if args:
42
            return runfcgi(func, validaddr(args[0]))
43
        else:
44
            return runfcgi(func, None)
45
    
5e-06 sec 46
    if 'scgi' in sys.argv:
47
        args = sys.argv[1:]
48
        args.remove('scgi')
49
        if args:
50
            return runscgi(func, validaddr(args[0]))
51
        else:
52
            return runscgi(func)
53
    
3e-06 sec 54
    return httpserver.runsimple(func, validip(listget(sys.argv, 1, '')))
55
    
56
def _is_dev_mode():
57
    # quick hack to check if the program is running in dev mode.
58
    if os.environ.has_key('SERVER_SOFTWARE') \
59
        or os.environ.has_key('PHP_FCGI_CHILDREN') \
60
        or 'fcgi' in sys.argv or 'fastcgi' in sys.argv:
61
            return False
62
    return True
63
64
# When running the builtin-server, enable debug mode if not already set.
65
web.config.setdefault('debug', _is_dev_mode())
/var/lib/python-support/python2.6/web/wsgiserver/__init__.py
     time      num code
1
"""A high-speed, production ready, thread pooled, generic WSGI server.
2
3
Simplest example on how to use this module directly
4
(without using CherryPy's application machinery):
5
6
    from cherrypy import wsgiserver
7
    
8
    def my_crazy_app(environ, start_response):
9
        status = '200 OK'
10
        response_headers = [('Content-type','text/plain')]
11
        start_response(status, response_headers)
12
        return ['Hello world!\n']
13
    
14
    server = wsgiserver.CherryPyWSGIServer(
15
                ('0.0.0.0', 8070), my_crazy_app,
16
                server_name='www.cherrypy.example')
17
    
18
The CherryPy WSGI server can serve as many WSGI applications 
19
as you want in one instance by using a WSGIPathInfoDispatcher:
20
    
21
    d = WSGIPathInfoDispatcher({'/': my_crazy_app, '/blog': my_blog_app})
22
    server = wsgiserver.CherryPyWSGIServer(('0.0.0.0', 80), d)
23
    
24
Want SSL support? Just set these attributes:
25
    
26
    server.ssl_certificate = <filename>
27
    server.ssl_private_key = <filename>
28
    
29
    if __name__ == '__main__':
30
        try:
31
            server.start()
32
        except KeyboardInterrupt:
33
            server.stop()
34
35
This won't call the CherryPy engine (application side) at all, only the
36
WSGI server, which is independant from the rest of CherryPy. Don't
37
let the name "CherryPyWSGIServer" throw you; the name merely reflects
38
its origin, not its coupling.
39
40
For those of you wanting to understand internals of this module, here's the
41
basic call flow. The server's listening thread runs a very tight loop,
42
sticking incoming connections onto a Queue:
43
44
    server = CherryPyWSGIServer(...)
45
    server.start()
46
    while True:
47
        tick()
48
        # This blocks until a request comes in:
49
        child = socket.accept()
50
        conn = HTTPConnection(child, ...)
51
        server.requests.put(conn)
52
53
Worker threads are kept in a pool and poll the Queue, popping off and then
54
handling each connection in turn. Each connection can consist of an arbitrary
55
number of requests and their responses, so we run a nested loop:
56
57
    while True:
58
        conn = server.requests.get()
59
        conn.communicate()
60
        ->  while True:
61
                req = HTTPRequest(...)
62
                req.parse_request()
63
                ->  # Read the Request-Line, e.g. "GET /page HTTP/1.1"
64
                    req.rfile.readline()
65
                    req.read_headers()
66
                req.respond()
67
                ->  response = wsgi_app(...)
68
                    try:
69
                        for chunk in response:
70
                            if chunk:
71
                                req.write(chunk)
72
                    finally:
73
                        if hasattr(response, "close"):
74
                            response.close()
75
                if req.close_connection:
76
                    return
0.001513 sec 77
"""
78
79
4e-06 sec 80
import base64
0.000113 sec 81
import os
5.9e-05 sec 82
import Queue
1.8e-05 sec 83
import re
7e-05 sec 84
quoted_slash = re.compile("(?i)%2F")
3e-06 sec 85
import rfc822
6.6e-05 sec 86
import socket
5.8e-05 sec 87
try:
2e-06 sec 88
    import cStringIO as StringIO
89
except ImportError:
90
    import StringIO
91
5.9e-05 sec 92
_fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring)
93
1e-05 sec 94
import sys
6.5e-05 sec 95
import threading
5.9e-05 sec 96
import time
6e-05 sec 97
import traceback
6.1e-05 sec 98
from urllib import unquote
6.7e-05 sec 99
from urlparse import urlparse
6.5e-05 sec 100
import warnings
101
6.5e-05 sec 102
try:
1e-06 sec 103
    from OpenSSL import SSL
2.2e-05 sec 104
    from OpenSSL import crypto
105
except ImportError:
106
    SSL = None
107
1.4e-05 sec 108
import errno
109
6.7e-05 sec 110
def plat_specific_errors(*errnames):
111
    """Return error numbers for all errors in errnames on this platform.
112
    
113
    The 'errno' module contains different global constants depending on
114
    the specific platform (OS). This function will return the list of
115
    numeric values for a given list of potential names.
116
    """
6e-06 sec 117
    errno_names = dir(errno)
0.00055 sec 118
    nums = [getattr(errno, k) for k in errnames if k in errno_names]
119
    # de-dupe the list
6e-06 sec 120
    return dict.fromkeys(nums).keys()
121
3e-06 sec 122
socket_error_eintr = plat_specific_errors("EINTR", "WSAEINTR")
123
4e-06 sec 124
socket_errors_to_ignore = plat_specific_errors(
2e-06 sec 125
    "EPIPE",
3e-06 sec 126
    "EBADF", "WSAEBADF",
2e-06 sec 127
    "ENOTSOCK", "WSAENOTSOCK",
2e-06 sec 128
    "ETIMEDOUT", "WSAETIMEDOUT",
2e-06 sec 129
    "ECONNREFUSED", "WSAECONNREFUSED",
1e-06 sec 130
    "ECONNRESET", "WSAECONNRESET",
2e-06 sec 131
    "ECONNABORTED", "WSAECONNABORTED",
2e-06 sec 132
    "ENETRESET", "WSAENETRESET",
3e-06 sec 133
    "EHOSTDOWN", "EHOSTUNREACH",
134
    )
5e-06 sec 135
socket_errors_to_ignore.append("timed out")
136
5e-06 sec 137
socket_errors_nonblocking = plat_specific_errors(
2e-06 sec 138
    'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK')
139
4e-06 sec 140
comma_separated_headers = ['ACCEPT', 'ACCEPT-CHARSET', 'ACCEPT-ENCODING',
1e-06 sec 141
    'ACCEPT-LANGUAGE', 'ACCEPT-RANGES', 'ALLOW', 'CACHE-CONTROL',
3e-06 sec 142
    'CONNECTION', 'CONTENT-ENCODING', 'CONTENT-LANGUAGE', 'EXPECT',
2e-06 sec 143
    'IF-MATCH', 'IF-NONE-MATCH', 'PRAGMA', 'PROXY-AUTHENTICATE', 'TE',
3e-06 sec 144
    'TRAILER', 'TRANSFER-ENCODING', 'UPGRADE', 'VARY', 'VIA', 'WARNING',
2e-06 sec 145
    'WWW-AUTHENTICATE']
146
147
1.1e-05 sec 148
class WSGIPathInfoDispatcher(object):
149
    """A WSGI dispatcher for dispatch based on the PATH_INFO.
150
    
151
    apps: a dict or list of (path_prefix, app) pairs.
2e-06 sec 152
    """
153
    
2e-06 sec 154
    def __init__(self, apps):
155
        try:
156
            apps = apps.items()
157
        except AttributeError:
158
            pass
159
        
160
        # Sort the apps by len(path), descending
161
        apps.sort()
162
        apps.reverse()
163
        
164
        # The path_prefix strings must start, but not end, with a slash.
165
        # Use "" instead of "/".
166
        self.apps = [(p.rstrip("/"), a) for p, a in apps]
167
    
2e-06 sec 168
    def __call__(self, environ, start_response):
169
        path = environ["PATH_INFO"] or "/"
170
        for p, app in self.apps:
171
            # The apps list should be sorted by length, descending.
172
            if path.startswith(p + "/") or path == p:
173
                environ = environ.copy()
174
                environ["SCRIPT_NAME"] = environ["SCRIPT_NAME"] + p
175
                environ["PATH_INFO"] = path[len(p):]
176
                return app(environ, start_response)
177
        
178
        start_response('404 Not Found', [('Content-Type', 'text/plain'),
179
                                         ('Content-Length', '0')])
180
        return ['']
181
182
8.8e-05 sec 183
class MaxSizeExceeded(Exception):
2e-06 sec 184
    pass
185
7.8e-05 sec 186
class SizeCheckWrapper(object):
3e-06 sec 187
    """Wraps a file-like object, raising MaxSizeExceeded if too large."""
188
    
1.1e-05 sec 189
    def __init__(self, rfile, maxlen):
2e-06 sec 190
        self.rfile = rfile
3e-06 sec 191
        self.maxlen = maxlen
4e-06 sec 192
        self.bytes_read = 0
193
    
2e-06 sec 194
    def _check_length(self):
195
        if self.maxlen and self.bytes_read > self.maxlen:
196
            raise MaxSizeExceeded()
197
    
2e-06 sec 198
    def read(self, size=None):
199
        data = self.rfile.read(size)
200
        self.bytes_read += len(data)
201
        self._check_length()
202
        return data
203
    
3e-06 sec 204
    def readline(self, size=None):
205
        if size is not None:
206
            data = self.rfile.readline(size)
207
            self.bytes_read += len(data)
208
            self._check_length()
209
            return data
210
        
211
        # User didn't specify a size ...
212
        # We read the line in chunks to make sure it's not a 100MB line !
213
        res = []
214
        while True:
215
            data = self.rfile.readline(256)
216
            self.bytes_read += len(data)
217
            self._check_length()
218
            res.append(data)
219
            # See http://www.cherrypy.org/ticket/421
220
            if len(data) < 256 or data[-1:] == "\n":
221
                return ''.join(res)
222
    
4e-06 sec 223
    def readlines(self, sizehint=0):
224
        # Shamelessly stolen from StringIO
225
        total = 0
226
        lines = []
227
        line = self.readline()
228
        while line:
229
            lines.append(line)
230
            total += len(line)
231
            if 0 < sizehint <= total:
232
                break
233
            line = self.readline()
234
        return lines
235
    
3e-06 sec 236
    def close(self):
237
        self.rfile.close()
238
    
3e-06 sec 239
    def __iter__(self):
240
        return self
241
    
2e-06 sec 242
    def next(self):
243
        data = self.rfile.next()
244
        self.bytes_read += len(data)
245
        self._check_length()
246
        return data
247
248
5.7e-05 sec 249
class HTTPRequest(object):
250
    """An HTTP Request (and response).
251
    
252
    A single HTTP connection may consist of multiple request/response pairs.
253
    
254
    send: the 'send' method from the connection's socket object.
255
    wsgi_app: the WSGI application to call.
256
    environ: a partial WSGI environ (server and connection entries).
257
        The caller MUST set the following entries:
258
        * All wsgi.* entries, including .input
259
        * SERVER_NAME and SERVER_PORT
260
        * Any SSL_* entries
261
        * Any custom entries like REMOTE_ADDR and REMOTE_PORT
262
        * SERVER_SOFTWARE: the value to write in the "Server" response header.
263
        * ACTUAL_SERVER_PROTOCOL: the value to write in the Status-Line of
264
            the response. From RFC 2145: "An HTTP server SHOULD send a
265
            response version equal to the highest version for which the
266
            server is at least conditionally compliant, and whose major
267
            version is less than or equal to the one received in the
268
            request.  An HTTP server MUST NOT send a version for which
269
            it is not at least conditionally compliant."
270
    
271
    outheaders: a list of header tuples to write in the response.
272
    ready: when True, the request has been parsed and is ready to begin
273
        generating the response. When False, signals the calling Connection
274
        that the response should not be generated and the connection should
275
        close.
276
    close_connection: signals the calling Connection that the request
277
        should close. This does not imply an error! The client and/or
278
        server may each request that the connection be closed.
279
    chunked_write: if True, output will be encoded with the "chunked"
280
        transfer-coding. This value is set automatically inside
281
        send_headers.
2e-06 sec 282
    """
283
    
1e-06 sec 284
    max_request_header_size = 0
2e-06 sec 285
    max_request_body_size = 0
286
    
3e-06 sec 287
    def __init__(self, wfile, environ, wsgi_app):
288
        self.rfile = environ['wsgi.input']
289
        self.wfile = wfile
290
        self.environ = environ.copy()
291
        self.wsgi_app = wsgi_app
292
        
293
        self.ready = False
294
        self.started_response = False
295
        self.status = ""
296
        self.outheaders = []
297
        self.sent_headers = False
298
        self.close_connection = False
299
        self.chunked_write = False
300
    
2e-06 sec 301
    def parse_request(self):
302
        """Parse the next HTTP request start-line and message-headers."""
303
        self.rfile.maxlen = self.max_request_header_size
304
        self.rfile.bytes_read = 0
305
        
306
        try:
307
            self._parse_request()
308
        except MaxSizeExceeded:
309
            self.simple_response("413 Request Entity Too Large")
310
            return
311
    
4e-06 sec 312
    def _parse_request(self):
313
        # HTTP/1.1 connections are persistent by default. If a client
314
        # requests a page, then idles (leaves the connection open),
315
        # then rfile.readline() will raise socket.error("timed out").
316
        # Note that it does this based on the value given to settimeout(),
317
        # and doesn't need the client to request or acknowledge the close
318
        # (although your TCP stack might suffer for it: cf Apache's history
319
        # with FIN_WAIT_2).
320
        request_line = self.rfile.readline()
321
        if not request_line:
322
            # Force self.ready = False so the connection will close.
323
            self.ready = False
324
            return
325
        
326
        if request_line == "\r\n":
327
            # RFC 2616 sec 4.1: "...if the server is reading the protocol
328
            # stream at the beginning of a message and receives a CRLF
329
            # first, it should ignore the CRLF."
330
            # But only ignore one leading line! else we enable a DoS.
331
            request_line = self.rfile.readline()
332
            if not request_line:
333
                self.ready = False
334
                return
335
        
336
        environ = self.environ
337
        
338
        try:
339
            method, path, req_protocol = request_line.strip().split(" ", 2)
340
        except ValueError:
341
            self.simple_response(400, "Malformed Request-Line")
342
            return
343
        
344
        environ["REQUEST_METHOD"] = method
345
        
346
        # path may be an abs_path (including "http://host.domain.tld");
347
        scheme, location, path, params, qs, frag = urlparse(path)
348
        
349
        if frag:
350
            self.simple_response("400 Bad Request",
351
                                 "Illegal #fragment in Request-URI.")
352
            return
353
        
354
        if scheme:
355
            environ["wsgi.url_scheme"] = scheme
356
        if params:
357
            path = path + ";" + params
358
        
359
        environ["SCRIPT_NAME"] = ""
360
        
361
        # Unquote the path+params (e.g. "/this%20path" -> "this path").
362
        # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2
363
        #
364
        # But note that "...a URI must be separated into its components
365
        # before the escaped characters within those components can be
366
        # safely decoded." http://www.ietf.org/rfc/rfc2396.txt, sec 2.4.2
367
        atoms = [unquote(x) for x in quoted_slash.split(path)]
368
        path = "%2F".join(atoms)
369
        environ["PATH_INFO"] = path
370
        
371
        # Note that, like wsgiref and most other WSGI servers,
372
        # we unquote the path but not the query string.
373
        environ["QUERY_STRING"] = qs
374
        
375
        # Compare request and server HTTP protocol versions, in case our
376
        # server does not support the requested protocol. Limit our output
377
        # to min(req, server). We want the following output:
378
        #     request    server     actual written   supported response
379
        #     protocol   protocol  response protocol    feature set
380
        # a     1.0        1.0           1.0                1.0
381
        # b     1.0        1.1           1.1                1.0
382
        # c     1.1        1.0           1.0                1.0
383
        # d     1.1        1.1           1.1                1.1
384
        # Notice that, in (b), the response will be "HTTP/1.1" even though
385
        # the client only understands 1.0. RFC 2616 10.5.6 says we should
386
        # only return 505 if the _major_ version is different.
387
        rp = int(req_protocol[5]), int(req_protocol[7])
388
        server_protocol = environ["ACTUAL_SERVER_PROTOCOL"]
389
        sp = int(server_protocol[5]), int(server_protocol[7])
390
        if sp[0] != rp[0]:
391
            self.simple_response("505 HTTP Version Not Supported")
392
            return
393
        # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol.
394
        environ["SERVER_PROTOCOL"] = req_protocol
395
        self.response_protocol = "HTTP/%s.%s" % min(rp, sp)
396
        
397
        # If the Request-URI was an absoluteURI, use its location atom.
398
        if location:
399
            environ["SERVER_NAME"] = location
400
        
401
        # then all the http headers
402
        try:
403
            self.read_headers()
404
        except ValueError, ex:
405
            self.simple_response("400 Bad Request", repr(ex.args))
406
            return
407
        
408
        mrbs = self.max_request_body_size
409
        if mrbs and int(environ.get("CONTENT_LENGTH", 0)) > mrbs:
410
            self.simple_response("413 Request Entity Too Large")
411
            return
412
        
413
        # Persistent connection support
414
        if self.response_protocol == "HTTP/1.1":
415
            # Both server and client are HTTP/1.1
416
            if environ.get("HTTP_CONNECTION", "") == "close":
417
                self.close_connection = True
418
        else:
419
            # Either the server or client (or both) are HTTP/1.0
420
            if environ.get("HTTP_CONNECTION", "") != "Keep-Alive":
421
                self.close_connection = True
422
        
423
        # Transfer-Encoding support
424
        te = None
425
        if self.response_protocol == "HTTP/1.1":
426
            te = environ.get("HTTP_TRANSFER_ENCODING")
427
            if te:
428
                te = [x.strip().lower() for x in te.split(",") if x.strip()]
429
        
430
        self.chunked_read = False
431
        
432
        if te:
433
            for enc in te:
434
                if enc == "chunked":
435
                    self.chunked_read = True
436
                else:
437
                    # Note that, even if we see "chunked", we must reject
438
                    # if there is an extension we don't recognize.
439
                    self.simple_response("501 Unimplemented")
440
                    self.close_connection = True
441
                    return
442
        
443
        # From PEP 333:
444
        # "Servers and gateways that implement HTTP 1.1 must provide
445
        # transparent support for HTTP 1.1's "expect/continue" mechanism.
446
        # This may be done in any of several ways:
447
        #   1. Respond to requests containing an Expect: 100-continue request
448
        #      with an immediate "100 Continue" response, and proceed normally.
449
        #   2. Proceed with the request normally, but provide the application
450
        #      with a wsgi.input stream that will send the "100 Continue"
451
        #      response if/when the application first attempts to read from
452
        #      the input stream. The read request must then remain blocked
453
        #      until the client responds.
454
        #   3. Wait until the client decides that the server does not support
455
        #      expect/continue, and sends the request body on its own.
456
        #      (This is suboptimal, and is not recommended.)
457
        #
458
        # We used to do 3, but are now doing 1. Maybe we'll do 2 someday,
459
        # but it seems like it would be a big slowdown for such a rare case.
460
        if environ.get("HTTP_EXPECT", "") == "100-continue":
461
            self.simple_response(100)
462
        
463
        self.ready = True
464
    
3e-06 sec 465
    def read_headers(self):
466
        """Read header lines from the incoming stream."""
467
        environ = self.environ
468
        
469
        while True:
470
            line = self.rfile.readline()
471
            if not line:
472
                # No more data--illegal end of headers
473
                raise ValueError("Illegal end of headers.")
474
            
475
            if line == '\r\n':
476
                # Normal end of headers
477
                break
478
            
479
            if line[0] in ' \t':
480
                # It's a continuation line.
481
                v = line.strip()
482
            else:
483
                k, v = line.split(":", 1)
484
                k, v = k.strip().upper(), v.strip()
485
                envname = "HTTP_" + k.replace("-", "_")
486
            
487
            if k in comma_separated_headers:
488
                existing = environ.get(envname)
489
                if existing:
490
                    v = ", ".join((existing, v))
491
            environ[envname] = v
492
        
493
        ct = environ.pop("HTTP_CONTENT_TYPE", None)
494
        if ct is not None:
495
            environ["CONTENT_TYPE"] = ct
496
        cl = environ.pop("HTTP_CONTENT_LENGTH", None)
497
        if cl is not None:
498
            environ["CONTENT_LENGTH"] = cl
499
    
3e-06 sec 500
    def decode_chunked(self):
501
        """Decode the 'chunked' transfer coding."""
502
        cl = 0
503
        data = StringIO.StringIO()
504
        while True:
505
            line = self.rfile.readline().strip().split(";", 1)
506
            chunk_size = int(line.pop(0), 16)
507
            if chunk_size <= 0:
508
                break
509
##            if line: chunk_extension = line[0]
510
            cl += chunk_size
511
            data.write(self.rfile.read(chunk_size))
512
            crlf = self.rfile.read(2)
513
            if crlf != "\r\n":
514
                self.simple_response("400 Bad Request",
515
                                     "Bad chunked transfer coding "
516
                                     "(expected '\\r\\n', got %r)" % crlf)
517
                return
518
        
519
        # Grab any trailer headers
520
        self.read_headers()
521
        
522
        data.seek(0)
523
        self.environ["wsgi.input"] = data
524
        self.environ["CONTENT_LENGTH"] = str(cl) or ""
525
        return True
526
    
3e-06 sec 527
    def respond(self):
528
        """Call the appropriate WSGI app and write its iterable output."""
529
        # Set rfile.maxlen to ensure we don't read past Content-Length.
530
        # This will also be used to read the entire request body if errors
531
        # are raised before the app can read the body.
532
        if self.chunked_read:
533
            # If chunked, Content-Length will be 0.
534
            self.rfile.maxlen = self.max_request_body_size
535
        else:
536
            cl = int(self.environ.get("CONTENT_LENGTH", 0))
537
            if self.max_request_body_size:
538
                self.rfile.maxlen = min(cl, self.max_request_body_size)
539
            else:
540
                self.rfile.maxlen = cl
541
        self.rfile.bytes_read = 0
542
        
543
        try:
544
            self._respond()
545
        except MaxSizeExceeded:
546
            if not self.sent_headers:
547
                self.simple_response("413 Request Entity Too Large")
548
            return
549
    
3e-06 sec 550
    def _respond(self):
551
        if self.chunked_read:
552
            if not self.decode_chunked():
553
                self.close_connection = True
554
                return
555
        
556
        response = self.wsgi_app(self.environ, self.start_response)
557
        try:
558
            for chunk in response:
559
                # "The start_response callable must not actually transmit
560
                # the response headers. Instead, it must store them for the
561
                # server or gateway to transmit only after the first
562
                # iteration of the application return value that yields
563
                # a NON-EMPTY string, or upon the application's first
564
                # invocation of the write() callable." (PEP 333)
565
                if chunk:
566
                    self.write(chunk)
567
        finally:
568
            if hasattr(response, "close"):
569
                response.close()
570
        
571
        if (self.ready and not self.sent_headers):
572
            self.sent_headers = True
573
            self.send_headers()
574
        if self.chunked_write:
575
            self.wfile.sendall("0\r\n\r\n")
576
    
3e-06 sec 577
    def simple_response(self, status, msg=""):
578
        """Write a simple response back to the client."""
579
        status = str(status)
580
        buf = ["%s %s\r\n" % (self.environ['ACTUAL_SERVER_PROTOCOL'], status),
581
               "Content-Length: %s\r\n" % len(msg),
582
               "Content-Type: text/plain\r\n"]
583
        
584
        if status[:3] == "413" and self.response_protocol == 'HTTP/1.1':
585
            # Request Entity Too Large
586
            self.close_connection = True
587
            buf.append("Connection: close\r\n")
588
        
589
        buf.append("\r\n")
590
        if msg:
591
            buf.append(msg)
592
        
593
        try:
594
            self.wfile.sendall("".join(buf))
595
        except socket.error, x:
596
            if x.args[0] not in socket_errors_to_ignore:
597
                raise
598
    
3e-06 sec 599
    def start_response(self, status, headers, exc_info = None):
600
        """WSGI callable to begin the HTTP response."""
601
        # "The application may call start_response more than once,
602
        # if and only if the exc_info argument is provided."
603
        if self.started_response and not exc_info:
604
            raise AssertionError("WSGI start_response called a second "
605
                                 "time with no exc_info.")
606
        
607
        # "if exc_info is provided, and the HTTP headers have already been
608
        # sent, start_response must raise an error, and should raise the
609
        # exc_info tuple."
610
        if self.sent_headers:
611
            try:
612
                raise exc_info[0], exc_info[1], exc_info[2]
613
            finally:
614
                exc_info = None
615
        
616
        self.started_response = True
617
        self.status = status
618
        self.outheaders.extend(headers)
619
        return self.write
620
    
2e-06 sec 621
    def write(self, chunk):
622
        """WSGI callable to write unbuffered data to the client.
623
        
624
        This method is also used internally by start_response (to write
625
        data from the iterable returned by the WSGI application).
626
        """
627
        if not self.started_response:
628
            raise AssertionError("WSGI write called before start_response.")
629
        
630
        if not self.sent_headers:
631
            self.sent_headers = True
632
            self.send_headers()
633
        
634
        if self.chunked_write and chunk:
635
            buf = [hex(len(chunk))[2:], "\r\n", chunk, "\r\n"]
636
            self.wfile.sendall("".join(buf))
637
        else:
638
            self.wfile.sendall(chunk)
639
    
3e-06 sec 640
    def send_headers(self):
641
        """Assert, process, and send the HTTP response message-headers."""
642
        hkeys = [key.lower() for key, value in self.outheaders]
643
        status = int(self.status[:3])
644
        
645
        if status == 413:
646
            # Request Entity Too Large. Close conn to avoid garbage.
647
            self.close_connection = True
648
        elif "content-length" not in hkeys:
649
            # "All 1xx (informational), 204 (no content),
650
            # and 304 (not modified) responses MUST NOT
651
            # include a message-body." So no point chunking.
652
            if status < 200 or status in (204, 205, 304):
653
                pass
654
            else:
655
                if (self.response_protocol == 'HTTP/1.1'
656
                    and self.environ["REQUEST_METHOD"] != 'HEAD'):
657
                    # Use the chunked transfer-coding
658
                    self.chunked_write = True
659
                    self.outheaders.append(("Transfer-Encoding", "chunked"))
660
                else:
661
                    # Closing the conn is the only way to determine len.
662
                    self.close_connection = True
663
        
664
        if "connection" not in hkeys:
665
            if self.response_protocol == 'HTTP/1.1':
666
                # Both server and client are HTTP/1.1 or better
667
                if self.close_connection:
668
                    self.outheaders.append(("Connection", "close"))
669
            else:
670
                # Server and/or client are HTTP/1.0
671
                if not self.close_connection:
672
                    self.outheaders.append(("Connection", "Keep-Alive"))
673
        
674
        if (not self.close_connection) and (not self.chunked_read):
675
            # Read any remaining request body data on the socket.
676
            # "If an origin server receives a request that does not include an
677
            # Expect request-header field with the "100-continue" expectation,
678
            # the request includes a request body, and the server responds
679
            # with a final status code before reading the entire request body
680
            # from the transport connection, then the server SHOULD NOT close
681
            # the transport connection until it has read the entire request,
682
            # or until the client closes the connection. Otherwise, the client
683
            # might not reliably receive the response message. However, this
684
            # requirement is not be construed as preventing a server from
685
            # defending itself against denial-of-service attacks, or from
686
            # badly broken client implementations."
687
            size = self.rfile.maxlen - self.rfile.bytes_read
688
            if size > 0:
689
                self.rfile.read(size)
690
        
691
        if "date" not in hkeys:
692
            self.outheaders.append(("Date", rfc822.formatdate()))
693
        
694
        if "server" not in hkeys:
695
            self.outheaders.append(("Server", self.environ['SERVER_SOFTWARE']))
696
        
697
        buf = [self.environ['ACTUAL_SERVER_PROTOCOL'], " ", self.status, "\r\n"]
698
        try:
699
            buf += [k + ": " + v + "\r\n" for k, v in self.outheaders]
700
        except TypeError:
701
            if not isinstance(k, str):
702
                raise TypeError("WSGI response header key %r is not a string.")
703
            if not isinstance(v, str):
704
                raise TypeError("WSGI response header value %r is not a string.")
705
            else:
706
                raise
707
        buf.append("\r\n")
708
        self.wfile.sendall("".join(buf))
709
710
6.3e-05 sec 711
class NoSSLError(Exception):
2e-06 sec 712
    """Exception raised when a client speaks HTTP to an HTTPS socket."""
2e-06 sec 713
    pass
714
715
6.7e-05 sec 716
class FatalSSLAlert(Exception):
2e-06 sec 717
    """Exception raised when the SSL implementation signals a fatal alert."""
2e-06 sec 718
    pass
719
720
6.7e-05 sec 721
if not _fileobject_uses_str_type:
1.1e-05 sec 722
    class CP_fileobject(socket._fileobject):
2e-06 sec 723
        """Faux file object attached to a socket object."""
724
2e-06 sec 725
        def sendall(self, data):
726
            """Sendall for non-blocking sockets."""
727
            while data:
728
                try:
729
                    bytes_sent = self.send(data)
730
                    data = data[bytes_sent:]
731
                except socket.error, e:
732
                    if e.args[0] not in socket_errors_nonblocking:
733
                        raise
734
3e-06 sec 735
        def send(self, data):
736
            return self._sock.send(data)
737
8e-06 sec 738
        def flush(self):
2e-06 sec 739
            if self._wbuf:
740
                buffer = "".join(self._wbuf)
741
                self._wbuf = []
742
                self.sendall(buffer)
743
3e-06 sec 744
        def recv(self, size):
745
            while True:
746
                try:
747
                    return self._sock.recv(size)
748
                except socket.error, e:
749
                    if (e.args[0] not in socket_errors_nonblocking
750
                        and e.args[0] not in socket_error_eintr):
751
                        raise
752
3e-06 sec 753
        def read(self, size=-1):
754
            # Use max, disallow tiny reads in a loop as they are very inefficient.
755
            # We never leave read() with any leftover data from a new recv() call
756
            # in our internal buffer.
757
            rbufsize = max(self._rbufsize, self.default_bufsize)
758
            # Our use of StringIO rather than lists of string objects returned by
759
            # recv() minimizes memory usage and fragmentation that occurs when
760
            # rbufsize is large compared to the typical return value of recv().
761
            buf = self._rbuf
762
            buf.seek(0, 2)  # seek end
763
            if size < 0:
764
                # Read until EOF
765
                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
766
                while True:
767
                    data = self.recv(rbufsize)
768
                    if not data:
769
                        break
770
                    buf.write(data)
771
                return buf.getvalue()
772
            else:
773
                # Read until size bytes or EOF seen, whichever comes first
774
                buf_len = buf.tell()
775
                if buf_len >= size:
776
                    # Already have size bytes in our buffer?  Extract and return.
777
                    buf.seek(0)
778
                    rv = buf.read(size)
779
                    self._rbuf = StringIO.StringIO()
780
                    self._rbuf.write(buf.read())
781
                    return rv
782
783
                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
784
                while True:
785
                    left = size - buf_len
786
                    # recv() will malloc the amount of memory given as its
787
                    # parameter even though it often returns much less data
788
                    # than that.  The returned data string is short lived
789
                    # as we copy it into a StringIO and free it.  This avoids
790
                    # fragmentation issues on many platforms.
791
                    data = self.recv(left)
792
                    if not data:
793
                        break
794
                    n = len(data)
795
                    if n == size and not buf_len:
796
                        # Shortcut.  Avoid buffer data copies when:
797
                        # - We have no data in our buffer.
798
                        # AND
799
                        # - Our call to recv returned exactly the
800
                        #   number of bytes we were asked to read.
801
                        return data
802
                    if n == left:
803
                        buf.write(data)
804
                        del data  # explicit free
805
                        break
806
                    assert n <= left, "recv(%d) returned %d bytes" % (left, n)
807
                    buf.write(data)
808
                    buf_len += n
809
                    del data  # explicit free
810
                    #assert buf_len == buf.tell()
811
                return buf.getvalue()
812
2e-06 sec 813
        def readline(self, size=-1):
814
            buf = self._rbuf
815
            buf.seek(0, 2)  # seek end
816
            if buf.tell() > 0:
817
                # check if we already have it in our buffer
818
                buf.seek(0)
819
                bline = buf.readline(size)
820
                if bline.endswith('\n') or len(bline) == size:
821
                    self._rbuf = StringIO.StringIO()
822
                    self._rbuf.write(buf.read())
823
                    return bline
824
                del bline
825
            if size < 0:
826
                # Read until \n or EOF, whichever comes first
827
                if self._rbufsize <= 1:
828
                    # Speed up unbuffered case
829
                    buf.seek(0)
830
                    buffers = [buf.read()]
831
                    self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
832
                    data = None
833
                    recv = self.recv
834
                    while data != "\n":
835
                        data = recv(1)
836
                        if not data:
837
                            break
838
                        buffers.append(data)
839
                    return "".join(buffers)
840
841
                buf.seek(0, 2)  # seek end
842
                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
843
                while True:
844
                    data = self.recv(self._rbufsize)
845
                    if not data:
846
                        break
847
                    nl = data.find('\n')
848
                    if nl >= 0:
849
                        nl += 1
850
                        buf.write(data[:nl])
851
                        self._rbuf.write(data[nl:])
852
                        del data
853
                        break
854
                    buf.write(data)
855
                return buf.getvalue()
856
            else:
857
                # Read until size bytes or \n or EOF seen, whichever comes first
858
                buf.seek(0, 2)  # seek end
859
                buf_len = buf.tell()
860
                if buf_len >= size:
861
                    buf.seek(0)
862
                    rv = buf.read(size)
863
                    self._rbuf = StringIO.StringIO()
864
                    self._rbuf.write(buf.read())
865
                    return rv
866
                self._rbuf = StringIO.StringIO()  # reset _rbuf.  we consume it via buf.
867
                while True:
868
                    data = self.recv(self._rbufsize)
869
                    if not data:
870
                        break
871
                    left = size - buf_len
872
                    # did we just receive a newline?
873
                    nl = data.find('\n', 0, left)
874
                    if nl >= 0:
875
                        nl += 1
876
                        # save the excess data to _rbuf
877
                        self._rbuf.write(data[nl:])
878
                        if buf_len:
879
                            buf.write(data[:nl])
880
                            break
881
                        else:
882
                            # Shortcut.  Avoid data copy through buf when returning
883
                            # a substring of our first recv().
884
                            return data[:nl]
885
                    n = len(data)
886
                    if n == size and not buf_len:
887
                        # Shortcut.  Avoid data copy through buf when
888
                        # returning exactly all of our first recv().
889
                        return data
890
                    if n >= left:
891
                        buf.write(data[:left])
892
                        self._rbuf.write(data[left:])
893
                        break
894
                    buf.write(data)
895
                    buf_len += n
896
                    #assert buf_len == buf.tell()
897
                return buf.getvalue()
898
899
else:
900
    class CP_fileobject(socket._fileobject):
901
        """Faux file object attached to a socket object."""
902
903
        def sendall(self, data):
904
            """Sendall for non-blocking sockets."""
905
            while data:
906
                try:
907
                    bytes_sent = self.send(data)
908
                    data = data[bytes_sent:]
909
                except socket.error, e:
910
                    if e.args[0] not in socket_errors_nonblocking:
911
                        raise
912
913
        def send(self, data):
914
            return self._sock.send(data)
915
916
        def flush(self):
917
            if self._wbuf:
918
                buffer = "".join(self._wbuf)
919
                self._wbuf = []
920
                self.sendall(buffer)
921
922
        def recv(self, size):
923
            while True:
924
                try:
925
                    return self._sock.recv(size)
926
                except socket.error, e:
927
                    if (e.args[0] not in socket_errors_nonblocking
928
                        and e.args[0] not in socket_error_eintr):
929
                        raise
930
931
        def read(self, size=-1):
932
            if size < 0:
933
                # Read until EOF
934
                buffers = [self._rbuf]
935
                self._rbuf = ""
936
                if self._rbufsize <= 1:
937
                    recv_size = self.default_bufsize
938
                else:
939
                    recv_size = self._rbufsize
940
941
                while True:
942
                    data = self.recv(recv_size)
943
                    if not data:
944
                        break
945
                    buffers.append(data)
946
                return "".join(buffers)
947
            else:
948
                # Read until size bytes or EOF seen, whichever comes first
949
                data = self._rbuf
950
                buf_len = len(data)
951
                if buf_len >= size:
952
                    self._rbuf = data[size:]
953
                    return data[:size]
954
                buffers = []
955
                if data:
956
                    buffers.append(data)
957
                self._rbuf = ""
958
                while True:
959
                    left = size - buf_len
960
                    recv_size = max(self._rbufsize, left)
961
                    data = self.recv(recv_size)
962
                    if not data:
963
                        break
964
                    buffers.append(data)
965
                    n = len(data)
966
                    if n >= left:
967
                        self._rbuf = data[left:]
968
                        buffers[-1] = data[:left]
969
                        break
970
                    buf_len += n
971
                return "".join(buffers)
972
973
        def readline(self, size=-1):
974
            data = self._rbuf
975
            if size < 0:
976
                # Read until \n or EOF, whichever comes first
977
                if self._rbufsize <= 1:
978
                    # Speed up unbuffered case
979
                    assert data == ""
980
                    buffers = []
981
                    while data != "\n":
982
                        data = self.recv(1)
983
                        if not data:
984
                            break
985
                        buffers.append(data)
986
                    return "".join(buffers)
987
                nl = data.find('\n')
988
                if nl >= 0:
989
                    nl += 1
990
                    self._rbuf = data[nl:]
991
                    return data[:nl]
992
                buffers = []
993
                if data:
994
                    buffers.append(data)
995
                self._rbuf = ""
996
                while True:
997
                    data = self.recv(self._rbufsize)
998
                    if not data:
999
                        break
1000
                    buffers.append(data)
1001
                    nl = data.find('\n')
1002
                    if nl >= 0:
1003
                        nl += 1
1004
                        self._rbuf = data[nl:]
1005
                        buffers[-1] = data[:nl]
1006
                        break
1007
                return "".join(buffers)
1008
            else:
1009
                # Read until size bytes or \n or EOF seen, whichever comes first
1010
                nl = data.find('\n', 0, size)
1011
                if nl >= 0:
1012
                    nl += 1
1013
                    self._rbuf = data[nl:]
1014
                    return data[:nl]
1015
                buf_len = len(data)
1016
                if buf_len >= size:
1017
                    self._rbuf = data[size:]
1018
                    return data[:size]
1019
                buffers = []
1020
                if data:
1021
                    buffers.append(data)
1022
                self._rbuf = ""
1023
                while True:
1024
                    data = self.recv(self._rbufsize)
1025
                    if not data:
1026
                        break
1027
                    buffers.append(data)
1028
                    left = size - buf_len
1029
                    nl = data.find('\n', 0, left)
1030
                    if nl >= 0:
1031
                        nl += 1
1032
                        self._rbuf = data[nl:]
1033
                        buffers[-1] = data[:nl]
1034
                        break
1035
                    n = len(data)
1036
                    if n >= left:
1037
                        self._rbuf = data[left:]
1038
                        buffers[-1] = data[:left]
1039
                        break
1040
                    buf_len += n
1041
                return "".join(buffers)
1042
    
1043
7e-05 sec 1044
class SSL_fileobject(CP_fileobject):
3e-06 sec 1045
    """SSL file object attached to a socket object."""
1046
    
1e-06 sec 1047
    ssl_timeout = 3
2e-06 sec 1048
    ssl_retry = .01
1049
    
2e-06 sec 1050
    def _safe_call(self, is_reader, call, *args, **kwargs):
1051
        """Wrap the given call with SSL error-trapping.
1052
        
1053
        is_reader: if False EOF errors will be raised. If True, EOF errors
1054
            will return "" (to emulate normal sockets).
1055
        """
1056
        start = time.time()
1057
        while True:
1058
            try:
1059
                return call(*args, **kwargs)
1060
            except SSL.WantReadError:
1061
                # Sleep and try again. This is dangerous, because it means
1062
                # the rest of the stack has no way of differentiating
1063
                # between a "new handshake" error and "client dropped".
1064
                # Note this isn't an endless loop: there's a timeout below.
1065
                time.sleep(self.ssl_retry)
1066
            except SSL.WantWriteError:
1067
                time.sleep(self.ssl_retry)
1068
            except SSL.SysCallError, e:
1069
                if is_reader and e.args == (-1, 'Unexpected EOF'):
1070
                    return ""
1071
                
1072
                errnum = e.args[0]
1073
                if is_reader and errnum in socket_errors_to_ignore:
1074
                    return ""
1075
                raise socket.error(errnum)
1076
            except SSL.Error, e:
1077
                if is_reader and e.args == (-1, 'Unexpected EOF'):
1078
                    return ""
1079
                
1080
                thirdarg = None
1081
                try:
1082
                    thirdarg = e.args[0][0][2]
1083
                except IndexError:
1084
                    pass
1085
                
1086
                if thirdarg == 'http request':
1087
                    # The client is talking HTTP to an HTTPS server.
1088
                    raise NoSSLError()
1089
                raise FatalSSLAlert(*e.args)
1090
            except:
1091
                raise
1092
            
1093
            if time.time() - start > self.ssl_timeout:
1094
                raise socket.timeout("timed out")
1095
3e-06 sec 1096
    def recv(self, *args, **kwargs):
1097
        buf = []
1098
        r = super(SSL_fileobject, self).recv
1099
        while True:
1100
            data = self._safe_call(True, r, *args, **kwargs)
1101
            buf.append(data)
1102
            p = self._sock.pending()
1103
            if not p:
1104
                return "".join(buf)
1105
    
3e-06 sec 1106
    def sendall(self, *args, **kwargs):
1107
        return self._safe_call(False, super(SSL_fileobject, self).sendall, *args, **kwargs)
1108
2e-06 sec 1109
    def send(self, *args, **kwargs):
1110
        return self._safe_call(False, super(SSL_fileobject, self).send, *args, **kwargs)
1111
1112
6.7e-05 sec 1113
class HTTPConnection(object):
1114
    """An HTTP connection (active socket).
1115
    
1116
    socket: the raw socket object (usually TCP) for this connection.
1117
    wsgi_app: the WSGI application for this server/connection.
1118
    environ: a WSGI environ template. This will be copied for each request.
1119
    
1120
    rfile: a fileobject for reading from the socket.
1121
    send: a function for writing (+ flush) to the socket.
2e-06 sec 1122
    """
1123
    
2e-06 sec 1124
    rbufsize = -1
3e-06 sec 1125
    RequestHandlerClass = HTTPRequest
2e-06 sec 1126
    environ = {"wsgi.version": (1, 0),
3e-06 sec 1127
               "wsgi.url_scheme": "http",
2e-06 sec 1128
               "wsgi.multithread": True,
3e-06 sec 1129
               "wsgi.multiprocess": False,
2e-06 sec 1130
               "wsgi.run_once": False,
2e-06 sec 1131
               "wsgi.errors": sys.stderr,
1132
               }
1133
    
1.4e-05 sec 1134
    def __init__(self, sock, wsgi_app, environ):
2e-06 sec 1135
        self.socket = sock
5e-06 sec 1136
        self.wsgi_app = wsgi_app
1137
        
1138
        # Copy the class environ into self.
4e-06 sec 1139
        self.environ = self.environ.copy()
7e-06 sec 1140
        self.environ.update(environ)
1141
        
1e-05 sec 1142
        if SSL and isinstance(sock, SSL.ConnectionType):
1143
            timeout = sock.gettimeout()
1144
            self.rfile = SSL_fileobject(sock, "rb", self.rbufsize)
1145
            self.rfile.ssl_timeout = timeout
1146
            self.wfile = SSL_fileobject(sock, "wb", -1)
1147
            self.wfile.ssl_timeout = timeout
1148
        else:
4.7e-05 sec 1149
            self.rfile = CP_fileobject(sock, "rb", self.rbufsize)
4e-06 sec 1150
            self.wfile = CP_fileobject(sock, "wb", -1)
1151
        
1152
        # Wrap wsgi.input but not HTTPConnection.rfile itself.
1153
        # We're also not setting maxlen yet; we'll do that separately
1154
        # for headers and body for each iteration of self.communicate
1155
        # (if maxlen is 0 the wrapper doesn't check length).
4e-06 sec 1156
        self.environ["wsgi.input"] = SizeCheckWrapper(self.rfile, 0)
1157
    
5e-06 sec 1158
    def communicate(self):
1159
        """Read each request and respond appropriately."""
1160
        try:
1161
            while True:
1162
                # (re)set req to None so that if something goes wrong in
1163
                # the RequestHandlerClass constructor, the error doesn't
1164
                # get written to the previous request.
1165
                req = None
1166
                req = self.RequestHandlerClass(self.wfile, self.environ,
1167
                                               self.wsgi_app)
1168
                
1169
                # This order of operations should guarantee correct pipelining.
1170
                req.parse_request()
1171
                if not req.ready:
1172
                    return
1173
                
1174
                req.respond()
1175
                if req.close_connection:
1176
                    return
1177
        
1178
        except socket.error, e:
1179
            errnum = e.args[0]
1180
            if errnum == 'timed out':
1181
                if req and not req.sent_headers:
1182
                    req.simple_response("408 Request Timeout")
1183
            elif errnum not in socket_errors_to_ignore:
1184
                if req and not req.sent_headers:
1185
                    req.simple_response("500 Internal Server Error",
1186
                                        format_exc())
1187
            return
1188
        except (KeyboardInterrupt, SystemExit):
1189
            raise
1190
        except FatalSSLAlert, e:
1191
            # Close the connection.
1192
            return
1193
        except NoSSLError:
1194
            # Unwrap our wfile
1195
            req.wfile = CP_fileobject(self.socket, "wb", -1)
1196
            if req and not req.sent_headers:
1197
                req.simple_response("400 Bad Request",
1198
                    "The client sent a plain HTTP request, but "
1199
                    "this server only speaks HTTPS on this port.")
1200
        except Exception, e:
1201
            if req and not req.sent_headers:
1202
                req.simple_response("500 Internal Server Error", format_exc())
1203
    
3e-06 sec 1204
    def close(self):
1205
        """Close the socket underlying this connection."""
1206
        self.rfile.close()
1207
        
1208
        # Python's socket module does NOT call close on the kernel socket
1209
        # when you call socket.close(). We do so manually here because we
1210
        # want this server to send a FIN TCP segment immediately. Note this
1211
        # must be called *before* calling socket.close(), because the latter
1212
        # drops its reference to the kernel socket.
1213
        self.socket._sock.close()
1214
        
1215
        self.socket.close()
1216
1217
5.5e-05 sec 1218
def format_exc(limit=None):
1219
    """Like print_exc() but return a string. Backport for Python 2.3."""
1220
    try:
1221
        etype, value, tb = sys.exc_info()
1222
        return ''.join(traceback.format_exception(etype, value, tb, limit))
1223
    finally:
1224
        etype = value = tb = None
1225
1226
4e-06 sec 1227
_SHUTDOWNREQUEST = None
1228
1.1e-05 sec 1229
class WorkerThread(threading.Thread):
1230
    """Thread which continuously polls a Queue for Connection objects.
1231
    
1232
    server: the HTTP Server which spawned this thread, and which owns the
1233
        Queue and is placing active connections into it.
1234
    ready: a simple flag for the calling server to know when this thread
1235
        has begun polling the Queue.
1236
    
1237
    Due to the timing issues of polling a Queue, a WorkerThread does not
1238
    check its own 'ready' flag after it has started. To stop the thread,
1239
    it is necessary to stick a _SHUTDOWNREQUEST object onto the Queue
1240
    (one for each running WorkerThread).
2e-06 sec 1241
    """
1242
    
2e-06 sec 1243
    conn = None
1244
    
8.3e-05 sec 1245
    def __init__(self, server):
1.7e-05 sec 1246
        self.ready = False
2.7e-05 sec 1247
        self.server = server
2.3e-05 sec 1248
        threading.Thread.__init__(self)
1249
    
3e-06 sec 1250
    def run(self):
1251
        try:
1252
            self.ready = True
1253
            while True:
1254
                conn = self.server.requests.get()
1255
                if conn is _SHUTDOWNREQUEST:
1256
                    return
1257
                
1258
                self.conn = conn
1259
                try:
1260
                    conn.communicate()
1261
                finally:
1262
                    conn.close()
1263
                    self.conn = None
1264
        except (KeyboardInterrupt, SystemExit), exc:
1265
            self.server.interrupt = exc
1266
1267
7e-05 sec 1268
class ThreadPool(object):
1269
    """A Request Queue for the CherryPyWSGIServer which pools threads.
1270
    
1271
    ThreadPool objects must provide min, get(), put(obj), start()
1272
    and stop(timeout) attributes.
2e-06 sec 1273
    """
1274
    
1.7e-05 sec 1275
    def __init__(self, server, min=10, max=-1):
2e-06 sec 1276
        self.server = server
5e-06 sec 1277
        self.min = min
2e-06 sec 1278
        self.max = max
3e-06 sec 1279
        self._threads = []
4e-06 sec 1280
        self._queue = Queue.Queue()
3e-06 sec 1281
        self.get = self._queue.get
1282
    
9e-06 sec 1283
    def start(self):
1284
        """Start the pool of threads."""
4e-05 sec 1285
        for i in xrange(self.min):
2.5e-05 sec 1286
            self._threads.append(WorkerThread(self.server))
2.8e-05 sec 1287
        for worker in self._threads:
2.3e-05 sec 1288
            worker.setName("CP WSGIServer " + worker.getName())
2.1e-05 sec 1289
            worker.start()
3.5e-05 sec 1290
        for worker in self._threads:
0.100191 sec 1291
            while not worker.ready:
3e-06 sec 1292
                time.sleep(.1)
1293
    
2e-06 sec 1294
    def _get_idle(self):
1295
        """Number of worker threads which are idle. Read-only."""
1296
        return len([t for t in self._threads if t.conn is None])
3e-06 sec 1297
    idle = property(_get_idle, doc=_get_idle.__doc__)
1298
    
1.3e-05 sec 1299
    def put(self, obj):
3e-06 sec 1300
        self._queue.put(obj)
1e-06 sec 1301
        if obj is _SHUTDOWNREQUEST:
1302
            return
1303
    
3e-06 sec 1304
    def grow(self, amount):
1305
        """Spawn new worker threads (not above self.max)."""
1306
        for i in xrange(amount):
1307
            if self.max > 0 and len(self._threads) >= self.max:
1308
                break
1309
            worker = WorkerThread(self.server)
1310
            worker.setName("CP WSGIServer " + worker.getName())
1311
            self._threads.append(worker)
1312
            worker.start()
1313
    
2e-06 sec 1314
    def shrink(self, amount):
1315
        """Kill off worker threads (not below self.min)."""
1316
        # Grow/shrink the pool if necessary.
1317
        # Remove any dead threads from our list
1318
        for t in self._threads:
1319
            if not t.isAlive():
1320
                self._threads.remove(t)
1321
                amount -= 1
1322
        
1323
        if amount > 0:
1324
            for i in xrange(min(amount, len(self._threads) - self.min)):
1325
                # Put a number of shutdown requests on the queue equal
1326
                # to 'amount'. Once each of those is processed by a worker,
1327
                # that worker will terminate and be culled from our list
1328
                # in self.put.
1329
                self._queue.put(_SHUTDOWNREQUEST)
1330
    
1.5e-05 sec 1331
    def stop(self, timeout=5):
1332
        # Must shut down threads here so the code that calls
1333
        # this method can know when all threads are stopped.
2.2e-05 sec 1334
        for worker in self._threads:
2.4e-05 sec 1335
            self._queue.put(_SHUTDOWNREQUEST)
1336
        
1337
        # Don't join currentThread (when stop is called inside a request).
2e-06 sec 1338
        current = threading.currentThread()
3e-05 sec 1339
        while self._threads:
2.1e-05 sec 1340
            worker = self._threads.pop()
7.2e-05 sec 1341
            if worker is not current and worker.isAlive():
4e-06 sec 1342
                try:
4e-06 sec 1343
                    if timeout is None or timeout < 0:
1344
                        worker.join()
1345
                    else:
6e-06 sec 1346
                        worker.join(timeout)
6e-06 sec 1347
                        if worker.isAlive():
1348
                            # We exhausted the timeout.
1349
                            # Forcibly shut down the socket.
2e-06 sec 1350
                            c = worker.conn
4e-06 sec 1351
                            if c and not c.rfile.closed:
2e-06 sec 1352
                                if SSL and isinstance(c.socket, SSL.ConnectionType):
1353
                                    # pyOpenSSL.socket.shutdown takes no args
1354
                                    c.socket.shutdown()
1355
                                else:
1.8e-05 sec 1356
                                    c.socket.shutdown(socket.SHUT_RD)
3e-06 sec 1357
                            worker.join()
1358
                except (AssertionError,
1359
                        # Ignore repeated Ctrl-C.
1360
                        # See http://www.cherrypy.org/ticket/691.
1361
                        KeyboardInterrupt), exc1:
1362
                    pass
1363
1364
1365
5.6e-05 sec 1366
class SSLConnection:
1367
    """A thread-safe wrapper for an SSL.Connection.
1368
    
1369
    *args: the arguments to create the wrapped SSL.Connection(*args).
2e-06 sec 1370
    """
1371
    
2e-06 sec 1372
    def __init__(self, *args):
1373
        self._ssl_conn = SSL.Connection(*args)
1374
        self._lock = threading.RLock()
1375
    
2e-06 sec 1376
    for f in ('get_context', 'pending', 'send', 'write', 'recv', 'read',
1377
              'renegotiate', 'bind', 'listen', 'connect', 'accept',
1378
              'setblocking', 'fileno', 'shutdown', 'close', 'get_cipher_list',
1379
              'getpeername', 'getsockname', 'getsockopt', 'setsockopt',
1380
              'makefile', 'get_app_data', 'set_app_data', 'state_string',
1381
              'sock_shutdown', 'get_peer_certificate', 'want_read',
1382
              'want_write', 'set_connect_state', 'set_accept_state',
0.000134 sec 1383
              'connect_ex', 'sendall', 'settimeout'):
1384
        exec """def %s(self, *args):
1385
        self._lock.acquire()
1386
        try:
1387
            return self._ssl_conn.%s(*args)
1388
        finally:
1389
            self._lock.release()
7.3e-05 sec 1390
""" % (f, f)
1391
1392
1e-05 sec 1393
try:
2e-06 sec 1394
    import fcntl
1395
except ImportError:
1396
    try:
1397
        from ctypes import windll, WinError
1398
    except ImportError:
1399
        def prevent_socket_inheritance(sock):
1400
            """Dummy function, since neither fcntl nor ctypes are available."""
1401
            pass
1402
    else:
1403
        def prevent_socket_inheritance(sock):
1404
            """Mark the given socket fd as non-inheritable (Windows)."""
1405
            if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0):
1406
                raise WinError()
1407
else:
8.2e-05 sec 1408
    def prevent_socket_inheritance(sock):
1409
        """Mark the given socket fd as non-inheritable (POSIX)."""
4e-06 sec 1410
        fd = sock.fileno()
4e-06 sec 1411
        old_flags = fcntl.fcntl(fd, fcntl.F_GETFD)
5.3e-05 sec 1412
        fcntl.fcntl(fd, fcntl.F_SETFD, old_flags | fcntl.FD_CLOEXEC)
1413
1414
1.1e-05 sec 1415
class CherryPyWSGIServer(object):
1416
    """An HTTP server for WSGI.
1417
    
1418
    bind_addr: The interface on which to listen for connections.
1419
        For TCP sockets, a (host, port) tuple. Host values may be any IPv4
1420
        or IPv6 address, or any valid hostname. The string 'localhost' is a
1421
        synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6).
1422
        The string '0.0.0.0' is a special IPv4 entry meaning "any active
1423
        interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for
1424
        IPv6. The empty string or None are not allowed.
1425
        
1426
        For UNIX sockets, supply the filename as a string.
1427
    wsgi_app: the WSGI 'application callable'; multiple WSGI applications
1428
        may be passed as (path_prefix, app) pairs.
1429
    numthreads: the number of worker threads to create (default 10).
1430
    server_name: the string to set for WSGI's SERVER_NAME environ entry.
1431
        Defaults to socket.gethostname().
1432
    max: the maximum number of queued requests (defaults to -1 = no limit).
1433
    request_queue_size: the 'backlog' argument to socket.listen();
1434
        specifies the maximum number of queued connections (default 5).
1435
    timeout: the timeout in seconds for accepted connections (default 10).
1436
    
1437
    nodelay: if True (the default since 3.1), sets the TCP_NODELAY socket
1438
        option.
1439
    
1440
    protocol: the version string to write in the Status-Line of all
1441
        HTTP responses. For example, "HTTP/1.1" (the default). This
1442
        also limits the supported features used in the response.
1443
    
1444
    
1445
    SSL/HTTPS
1446
    ---------
1447
    The OpenSSL module must be importable for SSL functionality.
1448
    You can obtain it from http://pyopenssl.sourceforge.net/
1449
    
1450
    ssl_certificate: the filename of the server SSL certificate.
1451
    ssl_privatekey: the filename of the server's private key file.
1452
    
1453
    If either of these is None (both are None by default), this server
1454
    will not use SSL. If both are given and are valid, they will be read
1455
    on server start and used in the SSL context for the listening socket.
3e-06 sec 1456
    """
1457
    
1e-06 sec 1458
    protocol = "HTTP/1.1"
3e-06 sec 1459
    _bind_addr = "127.0.0.1"
2e-06 sec 1460
    version = "CherryPy/3.1.0"
3e-06 sec 1461
    ready = False
3e-06 sec 1462
    _interrupt = None
1463
    
2e-06 sec 1464
    nodelay = True
1465
    
3e-06 sec 1466
    ConnectionClass = HTTPConnection
3e-06 sec 1467
    environ = {}
1468
    
1469
    # Paths to certificate and private key files
2e-06 sec 1470
    ssl_certificate = None
2e-06 sec 1471
    ssl_private_key = None
1472
    
1.4e-05 sec 1473
    def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None,
2e-06 sec 1474
                 max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5):
3e-06 sec 1475
        self.requests = ThreadPool(self, min=numthreads or 1, max=max)
1476
        
4e-06 sec 1477
        if callable(wsgi_app):
1478
            # We've been handed a single wsgi_app, in CP-2.1 style.
1479
            # Assume it's mounted at "".
6e-06 sec 1480
            self.wsgi_app = wsgi_app
1481
        else:
1482
            # We've been handed a list of (path_prefix, wsgi_app) tuples,
1483
            # so that the server can call different wsgi_apps, and also
1484
            # correctly set SCRIPT_NAME.
1485
            warnings.warn("The ability to pass multiple apps is deprecated "
1486
                          "and will be removed in 3.2. You should explicitly "
1487
                          "include a WSGIPathInfoDispatcher instead.",
1488
                          DeprecationWarning)
1489
            self.wsgi_app = WSGIPathInfoDispatcher(wsgi_app)
1490
        
3e-06 sec 1491
        self.bind_addr = bind_addr
2e-06 sec 1492
        if not server_name:
1493
            server_name = socket.gethostname()
2e-06 sec 1494
        self.server_name = server_name
3e-06 sec 1495
        self.request_queue_size = request_queue_size
1496
        
2e-06 sec 1497
        self.timeout = timeout
5e-06 sec 1498
        self.shutdown_timeout = shutdown_timeout
1499
    
2e-06 sec 1500
    def _get_numthreads(self):
1501
        return self.requests.min
3e-06 sec 1502
    def _set_numthreads(self, value):
1503
        self.requests.min = value
3e-06 sec 1504
    numthreads = property(_get_numthreads, _set_numthreads)
1505
    
6e-06 sec 1506
    def __str__(self):
1507
        return "%s.%s(%r)" % (self.__module__, self.__class__.__name__,
1508
                              self.bind_addr)
1509
    
4.8e-05 sec 1510
    def _get_bind_addr(self):
1.3e-05 sec 1511
        return self._bind_addr
1.1e-05 sec 1512
    def _set_bind_addr(self, value):
2e-06 sec 1513
        if isinstance(value, tuple) and value[0] in ('', None):
1514
            # Despite the socket module docs, using '' does not
1515
            # allow AI_PASSIVE to work. Passing None instead
1516
            # returns '0.0.0.0' like we want. In other words:
1517
            #     host    AI_PASSIVE     result
1518
            #      ''         Y         192.168.x.y
1519
            #      ''         N         192.168.x.y
1520
            #     None        Y         0.0.0.0
1521
            #     None        N         127.0.0.1
1522
            # But since you can get the same effect with an explicit
1523
            # '0.0.0.0', we deny both the empty string and None as values.
1524
            raise ValueError("Host values of '' or None are not allowed. "
1525
                             "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead "
1526
                             "to listen on all active interfaces.")
9e-06 sec 1527
        self._bind_addr = value
2e-06 sec 1528
    bind_addr = property(_get_bind_addr, _set_bind_addr,
1529
        doc="""The interface on which to listen for connections.
1530
        
1531
        For TCP sockets, a (host, port) tuple. Host values may be any IPv4
1532
        or IPv6 address, or any valid hostname. The string 'localhost' is a
1533
        synonym for '127.0.0.1' (or '::1', if your hosts file prefers IPv6).
1534
        The string '0.0.0.0' is a special IPv4 entry meaning "any active
1535
        interface" (INADDR_ANY), and '::' is the similar IN6ADDR_ANY for
1536
        IPv6. The empty string or None are not allowed.
1537
        
2e-06 sec 1538
        For UNIX sockets, supply the filename as a string.""")
1539
    
1.4e-05 sec 1540
    def start(self):
1541
        """Run the server forever."""
1542
        # We don't have to trap KeyboardInterrupt or SystemExit here,
1543
        # because cherrpy.server already does so, calling self.stop() for us.
1544
        # If you're using this server with another framework, you should
1545
        # trap those exceptions in whatever code block calls start().
2e-06 sec 1546
        self._interrupt = None
1547
        
1548
        # Select the appropriate socket
3e-06 sec 1549
        if isinstance(self.bind_addr, basestring):
1550
            # AF_UNIX socket
1551
            
1552
            # So we can reuse the socket...
1553
            try: os.unlink(self.bind_addr)
1554
            except: pass
1555
            
1556
            # So everyone can access the socket...
1557
            try: os.chmod(self.bind_addr, 0777)
1558
            except: pass
1559
            
1560
            info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)]
1561
        else:
1562
            # AF_INET or AF_INET6 socket
1563
            # Get the correct address family for our host (allows IPv6 addresses)
8e-06 sec 1564
            host, port = self.bind_addr
2e-06 sec 1565
            try:
2e-06 sec 1566
                info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
5e-06 sec 1567
                                          socket.SOCK_STREAM, 0, socket.AI_PASSIVE)
1568
            except socket.gaierror:
1569
                # Probably a DNS issue. Assume IPv4.
1570
                info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", self.bind_addr)]
1571
        
0.000191 sec 1572
        self.socket = None
4e-06 sec 1573
        msg = "No socket could be created"
2e-06 sec 1574
        for res in info:
3e-06 sec 1575
            af, socktype, proto, canonname, sa = res
4e-06 sec 1576
            try:
2e-06 sec 1577
                self.bind(af, socktype, proto)
1578
            except socket.error, msg:
1579
                if self.socket:
1580
                    self.socket.close()
1581
                self.socket = None
1582
                continue
3e-06 sec 1583
            break
2e-06 sec 1584
        if not self.socket:
1585
            raise socket.error, msg
1586
        
1587
        # Timeout so KeyboardInterrupt can be caught on Win32
4e-06 sec 1588
        self.socket.settimeout(1)
2e-06 sec 1589
        self.socket.listen(self.request_queue_size)
1590
        
1591
        # Create worker threads
3e-06 sec 1592
        self.requests.start()
1593
        
4e-06 sec 1594
        self.ready = True
3.8e-05 sec 1595
        while self.ready:
2.6e-05 sec 1596
            self.tick()
7.8e-05 sec 1597
            if self.interrupt:
1598
                while self.interrupt is True:
1599
                    # Wait for self.stop() to complete. See _set_interrupt.
1600
                    time.sleep(0.1)
1601
                if self.interrupt:
1602
                    raise self.interrupt
1603
    
1e-05 sec 1604
    def bind(self, family, type, proto=0):
1605
        """Create (or recreate) the actual socket object."""
2e-06 sec 1606
        self.socket = socket.socket(family, type, proto)
4e-06 sec 1607
        prevent_socket_inheritance(self.socket)
2e-06 sec 1608
        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
2e-06 sec 1609
        if self.nodelay:
4e-06 sec 1610
            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
2e-06 sec 1611
        if self.ssl_certificate and self.ssl_private_key:
1612
            if SSL is None:
1613
                raise ImportError("You must install pyOpenSSL to use HTTPS.")
1614
            
1615
            # See http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/442473
1616
            ctx = SSL.Context(SSL.SSLv23_METHOD)
1617
            ctx.use_privatekey_file(self.ssl_private_key)
1618
            ctx.use_certificate_file(self.ssl_certificate)
1619
            self.socket = SSLConnection(ctx, self.socket)
1620
            self.populate_ssl_environ()
1621
            
1622
            # If listening on the IPV6 any address ('::' = IN6ADDR_ANY),
1623
            # activate dual-stack. See http://www.cherrypy.org/ticket/871.
1624
            if (not isinstance(self.bind_addr, basestring)
1625
                and self.bind_addr[0] == '::' and family == socket.AF_INET6):
1626
                try:
1627
                    self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
1628
                except (AttributeError, socket.error):
1629
                    # Apparently, the socket option is not available in
1630
                    # this machine's TCP stack
1631
                    pass
1632
        
4e-06 sec 1633
        self.socket.bind(self.bind_addr)
1634
    
7.3e-05 sec 1635
    def tick(self):
1636
        """Accept a new connection and put it on the Queue."""
1.4e-05 sec 1637
        try:
2.1e-05 sec 1638
            s, addr = self.socket.accept()
5e-06 sec 1639
            prevent_socket_inheritance(s)
2e-06 sec 1640
            if not self.ready:
1641
                return
3e-06 sec 1642
            if hasattr(s, 'settimeout'):
6e-06 sec 1643
                s.settimeout(self.timeout)
1644
            
3e-06 sec 1645
            environ = self.environ.copy()
1646
            # SERVER_SOFTWARE is common for IIS. It's also helpful for
1647
            # us to pass a default value for the "Server" response header.
6e-06 sec 1648
            if environ.get("SERVER_SOFTWARE") is None:
5e-06 sec 1649
                environ["SERVER_SOFTWARE"] = "%s WSGI Server" % self.version
1650
            # set a non-standard environ entry so the WSGI app can know what
1651
            # the *real* server protocol is (and what features to support).
1652
            # See http://www.faqs.org/rfcs/rfc2145.html.
1.4e-05 sec 1653
            environ["ACTUAL_SERVER_PROTOCOL"] = self.protocol
4e-06 sec 1654
            environ["SERVER_NAME"] = self.server_name
1655
            
3e-06 sec 1656
            if isinstance(self.bind_addr, basestring):
1657
                # AF_UNIX. This isn't really allowed by WSGI, which doesn't
1658
                # address unix domain sockets. But it's better than nothing.
1659
                environ["SERVER_PORT"] = ""
1660
            else:
1.1e-05 sec 1661
                environ["SERVER_PORT"] = str(self.bind_addr[1])
1662
                # optional values
1663
                # Until we do DNS lookups, omit REMOTE_HOST
5e-05 sec 1664
                environ["REMOTE_ADDR"] = addr[0]
2e-06 sec 1665
                environ["REMOTE_PORT"] = str(addr[1])
1666
            
3.1e-05 sec 1667
            conn = self.ConnectionClass(s, self.wsgi_app, environ)
3e-06 sec 1668
            self.requests.put(conn)
0.000146 sec 1669
        except socket.timeout:
1670
            # The only reason for the timeout in start() is so we can
1671
            # notice keyboard interrupts on Win32, which don't interrupt
1672
            # accept() by default
7.5e-05 sec 1673
            return
7e-06 sec 1674
        except socket.error, x:
1675
            if x.args[0] in socket_error_eintr:
1676
                # I *think* this is right. EINTR should occur when a signal
1677
                # is received during the accept() call; all docs say retry
1678
                # the call, and I *think* I'm reading it right that Python
1679
                # will then go ahead and poll for and handle the signal
1680
                # elsewhere. See http://www.cherrypy.org/ticket/707.
1681
                return
1682
            if x.args[0] in socket_errors_nonblocking:
1683
                # Just try again. See http://www.cherrypy.org/ticket/479.
1684
                return
1685
            if x.args[0] in socket_errors_to_ignore:
1686
                # Our socket was closed.
1687
                # See http://www.cherrypy.org/ticket/686.
1688
                return
1689
            raise
1690
    
0.000106 sec 1691
    def _get_interrupt(self):
1.8e-05 sec 1692
        return self._interrupt
3e-06 sec 1693
    def _set_interrupt(self, interrupt):
1694
        self._interrupt = True
1695
        self.stop()
1696
        self._interrupt = interrupt
2e-06 sec 1697
    interrupt = property(_get_interrupt, _set_interrupt,
3e-06 sec 1698
                         doc="Set this to an Exception instance to "
1699
                             "interrupt the server.")
1700
    
4.7e-05 sec 1701
    def stop(self):
1702
        """Gracefully shutdown a server that is serving forever."""
2e-06 sec 1703
        self.ready = False
1704
        
5e-06 sec 1705
        sock = getattr(self, "socket", None)
5e-06 sec 1706
        if sock:
3e-06 sec 1707
            if not isinstance(self.bind_addr, basestring):
1708
                # Touch our own socket to make accept() return immediately.
1.1e-05 sec 1709
                try:
2e-06 sec 1710
                    host, port = sock.getsockname()[:2]
1711
                except socket.error, x:
1712
                    if x.args[1] != "Bad file descriptor":
1713
                        raise
1714
                else:
1715
                    # Note that we're explicitly NOT using AI_PASSIVE,
1716
                    # here, because we want an actual IP to touch.
1717
                    # localhost won't work if we've bound to a public IP,
1718
                    # but it will if we bound to '0.0.0.0' (INADDR_ANY).
6e-06 sec 1719
                    for res in socket.getaddrinfo(host, port, socket.AF_UNSPEC,
1e-05 sec 1720
                                                  socket.SOCK_STREAM):
0.000122 sec 1721
                        af, socktype, proto, canonname, sa = res
2e-06 sec 1722
                        s = None
2e-06 sec 1723
                        try:
2e-06 sec 1724
                            s = socket.socket(af, socktype, proto)
1725
                            # See http://groups.google.com/group/cherrypy-users/
1726
                            #        browse_frm/thread/bbfe5eb39c904fe0
4e-06 sec 1727
                            s.settimeout(1.0)
2e-06 sec 1728
                            s.connect((host, port))
4e-06 sec 1729
                            s.close()
1730
                        except socket.error:
1731
                            if s:
1732
                                s.close()
4e-06 sec 1733
            if hasattr(sock, "close"):
5e-06 sec 1734
                sock.close()
2e-06 sec 1735
            self.socket = None
1736
        
3e-06 sec 1737
        self.requests.stop(self.shutdown_timeout)
1738
    
2e-06 sec 1739
    def populate_ssl_environ(self):
1740
        """Create WSGI environ entries to be merged into each request."""
1741
        cert = open(self.ssl_certificate, 'rb').read()
1742
        cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert)
1743
        ssl_environ = {
1744
            "wsgi.url_scheme": "https",
1745
            "HTTPS": "on",
1746
            # pyOpenSSL doesn't provide access to any of these AFAICT
1747
##            'SSL_PROTOCOL': 'SSLv2',
1748
##            SSL_CIPHER 	string 	The cipher specification name
1749
##            SSL_VERSION_INTERFACE 	string 	The mod_ssl program version
1750
##            SSL_VERSION_LIBRARY 	string 	The OpenSSL program version
1751
            }
1752
        
1753
        # Server certificate attributes
1754
        ssl_environ.update({
1755
            'SSL_SERVER_M_VERSION': cert.get_version(),
1756
            'SSL_SERVER_M_SERIAL': cert.get_serial_number(),
1757
##            'SSL_SERVER_V_START': Validity of server's certificate (start time),
1758
##            'SSL_SERVER_V_END': Validity of server's certificate (end time),
1759
            })
1760
        
1761
        for prefix, dn in [("I", cert.get_issuer()),
1762
                           ("S", cert.get_subject())]:
1763
            # X509Name objects don't seem to have a way to get the
1764
            # complete DN string. Use str() and slice it instead,
1765
            # because str(dn) == "<X509Name object '/C=US/ST=...'>"
1766
            dnstr = str(dn)[18:-2]
1767
            
1768
            wsgikey = 'SSL_SERVER_%s_DN' % prefix
1769
            ssl_environ[wsgikey] = dnstr
1770
            
1771
            # The DN should be of the form: /k1=v1/k2=v2, but we must allow
1772
            # for any value to contain slashes itself (in a URL).
1773
            while dnstr:
1774
                pos = dnstr.rfind("=")
1775
                dnstr, value = dnstr[:pos], dnstr[pos + 1:]
1776
                pos = dnstr.rfind("/")
1777
                dnstr, key = dnstr[:pos], dnstr[pos + 1:]
1778
                if key and value:
1779
                    wsgikey = 'SSL_SERVER_%s_DN_%s' % (prefix, key)
1780
                    ssl_environ[wsgikey] = value
1781
        
1782
        self.environ.update(ssl_environ)