1"""An NNTP client class based on: 2- RFC 977: Network News Transfer Protocol 3- RFC 2980: Common NNTP Extensions 4- RFC 3977: Network News Transfer Protocol (version 2) 5 6Example: 7 8>>> from nntplib import NNTP 9>>> s = NNTP('news') 10>>> resp, count, first, last, name = s.group('comp.lang.python') 11>>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) 12Group comp.lang.python has 51 articles, range 5770 to 5821 13>>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last)) 14>>> resp = s.quit() 15>>> 16 17Here 'resp' is the server response line. 18Error responses are turned into exceptions. 19 20To post an article from a file: 21>>> f = open(filename, 'rb') # file containing article, including header 22>>> resp = s.post(f) 23>>> 24 25For descriptions of all methods, read the comments in the code below. 26Note that all arguments and return values representing article numbers 27are strings, not numbers, since they are rarely used for calculations. 28""" 29 30# RFC 977 by Brian Kantor and Phil Lapsley. 31# xover, xgtitle, xpath, date methods by Kevan Heydon 32 33# Incompatible changes from the 2.x nntplib: 34# - all commands are encoded as UTF-8 data (using the "surrogateescape" 35# error handler), except for raw message data (POST, IHAVE) 36# - all responses are decoded as UTF-8 data (using the "surrogateescape" 37# error handler), except for raw message data (ARTICLE, HEAD, BODY) 38# - the `file` argument to various methods is keyword-only 39# 40# - NNTP.date() returns a datetime object 41# - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object, 42# rather than a pair of (date, time) strings. 43# - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples 44# - NNTP.descriptions() returns a dict mapping group names to descriptions 45# - NNTP.xover() returns a list of dicts mapping field names (header or metadata) 46# to field values; each dict representing a message overview. 47# - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo) 48# tuple. 49# - the "internal" methods have been marked private (they now start with 50# an underscore) 51 52# Other changes from the 2.x/3.1 nntplib: 53# - automatic querying of capabilities at connect 54# - New method NNTP.getcapabilities() 55# - New method NNTP.over() 56# - New helper function decode_header() 57# - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and 58# arbitrary iterables yielding lines. 59# - An extensive test suite :-) 60 61# TODO: 62# - return structured data (GroupInfo etc.) everywhere 63# - support HDR 64 65# Imports 66import re 67import socket 68import collections 69import datetime 70import sys 71 72try: 73 import ssl 74except ImportError: 75 _have_ssl = False 76else: 77 _have_ssl = True 78 79from email.header import decode_header as _email_decode_header 80from socket import _GLOBAL_DEFAULT_TIMEOUT 81 82__all__ = ["NNTP", 83 "NNTPError", "NNTPReplyError", "NNTPTemporaryError", 84 "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError", 85 "decode_header", 86 ] 87 88# maximal line length when calling readline(). This is to prevent 89# reading arbitrary length lines. RFC 3977 limits NNTP line length to 90# 512 characters, including CRLF. We have selected 2048 just to be on 91# the safe side. 92_MAXLINE = 2048 93 94 95# Exceptions raised when an error or invalid response is received 96class NNTPError(Exception): 97 """Base class for all nntplib exceptions""" 98 def __init__(self, *args): 99 Exception.__init__(self, *args) 100 try: 101 self.response = args[0] 102 except IndexError: 103 self.response = 'No response given' 104 105class NNTPReplyError(NNTPError): 106 """Unexpected [123]xx reply""" 107 pass 108 109class NNTPTemporaryError(NNTPError): 110 """4xx errors""" 111 pass 112 113class NNTPPermanentError(NNTPError): 114 """5xx errors""" 115 pass 116 117class NNTPProtocolError(NNTPError): 118 """Response does not begin with [1-5]""" 119 pass 120 121class NNTPDataError(NNTPError): 122 """Error in response data""" 123 pass 124 125 126# Standard port used by NNTP servers 127NNTP_PORT = 119 128NNTP_SSL_PORT = 563 129 130# Response numbers that are followed by additional text (e.g. article) 131_LONGRESP = { 132 '100', # HELP 133 '101', # CAPABILITIES 134 '211', # LISTGROUP (also not multi-line with GROUP) 135 '215', # LIST 136 '220', # ARTICLE 137 '221', # HEAD, XHDR 138 '222', # BODY 139 '224', # OVER, XOVER 140 '225', # HDR 141 '230', # NEWNEWS 142 '231', # NEWGROUPS 143 '282', # XGTITLE 144} 145 146# Default decoded value for LIST OVERVIEW.FMT if not supported 147_DEFAULT_OVERVIEW_FMT = [ 148 "subject", "from", "date", "message-id", "references", ":bytes", ":lines"] 149 150# Alternative names allowed in LIST OVERVIEW.FMT response 151_OVERVIEW_FMT_ALTERNATIVES = { 152 'bytes': ':bytes', 153 'lines': ':lines', 154} 155 156# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) 157_CRLF = b'\r\n' 158 159GroupInfo = collections.namedtuple('GroupInfo', 160 ['group', 'last', 'first', 'flag']) 161 162ArticleInfo = collections.namedtuple('ArticleInfo', 163 ['number', 'message_id', 'lines']) 164 165 166# Helper function(s) 167def decode_header(header_str): 168 """Takes a unicode string representing a munged header value 169 and decodes it as a (possibly non-ASCII) readable value.""" 170 parts = [] 171 for v, enc in _email_decode_header(header_str): 172 if isinstance(v, bytes): 173 parts.append(v.decode(enc or 'ascii')) 174 else: 175 parts.append(v) 176 return ''.join(parts) 177 178def _parse_overview_fmt(lines): 179 """Parse a list of string representing the response to LIST OVERVIEW.FMT 180 and return a list of header/metadata names. 181 Raises NNTPDataError if the response is not compliant 182 (cf. RFC 3977, section 8.4).""" 183 fmt = [] 184 for line in lines: 185 if line[0] == ':': 186 # Metadata name (e.g. ":bytes") 187 name, _, suffix = line[1:].partition(':') 188 name = ':' + name 189 else: 190 # Header name (e.g. "Subject:" or "Xref:full") 191 name, _, suffix = line.partition(':') 192 name = name.lower() 193 name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name) 194 # Should we do something with the suffix? 195 fmt.append(name) 196 defaults = _DEFAULT_OVERVIEW_FMT 197 if len(fmt) < len(defaults): 198 raise NNTPDataError("LIST OVERVIEW.FMT response too short") 199 if fmt[:len(defaults)] != defaults: 200 raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields") 201 return fmt 202 203def _parse_overview(lines, fmt, data_process_func=None): 204 """Parse the response to an OVER or XOVER command according to the 205 overview format `fmt`.""" 206 n_defaults = len(_DEFAULT_OVERVIEW_FMT) 207 overview = [] 208 for line in lines: 209 fields = {} 210 article_number, *tokens = line.split('\t') 211 article_number = int(article_number) 212 for i, token in enumerate(tokens): 213 if i >= len(fmt): 214 # XXX should we raise an error? Some servers might not 215 # support LIST OVERVIEW.FMT and still return additional 216 # headers. 217 continue 218 field_name = fmt[i] 219 is_metadata = field_name.startswith(':') 220 if i >= n_defaults and not is_metadata: 221 # Non-default header names are included in full in the response 222 # (unless the field is totally empty) 223 h = field_name + ": " 224 if token and token[:len(h)].lower() != h: 225 raise NNTPDataError("OVER/XOVER response doesn't include " 226 "names of additional headers") 227 token = token[len(h):] if token else None 228 fields[fmt[i]] = token 229 overview.append((article_number, fields)) 230 return overview 231 232def _parse_datetime(date_str, time_str=None): 233 """Parse a pair of (date, time) strings, and return a datetime object. 234 If only the date is given, it is assumed to be date and time 235 concatenated together (e.g. response to the DATE command). 236 """ 237 if time_str is None: 238 time_str = date_str[-6:] 239 date_str = date_str[:-6] 240 hours = int(time_str[:2]) 241 minutes = int(time_str[2:4]) 242 seconds = int(time_str[4:]) 243 year = int(date_str[:-4]) 244 month = int(date_str[-4:-2]) 245 day = int(date_str[-2:]) 246 # RFC 3977 doesn't say how to interpret 2-char years. Assume that 247 # there are no dates before 1970 on Usenet. 248 if year < 70: 249 year += 2000 250 elif year < 100: 251 year += 1900 252 return datetime.datetime(year, month, day, hours, minutes, seconds) 253 254def _unparse_datetime(dt, legacy=False): 255 """Format a date or datetime object as a pair of (date, time) strings 256 in the format required by the NEWNEWS and NEWGROUPS commands. If a 257 date object is passed, the time is assumed to be midnight (00h00). 258 259 The returned representation depends on the legacy flag: 260 * if legacy is False (the default): 261 date has the YYYYMMDD format and time the HHMMSS format 262 * if legacy is True: 263 date has the YYMMDD format and time the HHMMSS format. 264 RFC 3977 compliant servers should understand both formats; therefore, 265 legacy is only needed when talking to old servers. 266 """ 267 if not isinstance(dt, datetime.datetime): 268 time_str = "000000" 269 else: 270 time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt) 271 y = dt.year 272 if legacy: 273 y = y % 100 274 date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt) 275 else: 276 date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt) 277 return date_str, time_str 278 279 280if _have_ssl: 281 282 def _encrypt_on(sock, context, hostname): 283 """Wrap a socket in SSL/TLS. Arguments: 284 - sock: Socket to wrap 285 - context: SSL context to use for the encrypted connection 286 Returns: 287 - sock: New, encrypted socket. 288 """ 289 # Generate a default SSL context if none was passed. 290 if context is None: 291 context = ssl._create_stdlib_context() 292 return context.wrap_socket(sock, server_hostname=hostname) 293 294 295# The classes themselves 296class NNTP: 297 # UTF-8 is the character set for all NNTP commands and responses: they 298 # are automatically encoded (when sending) and decoded (and receiving) 299 # by this class. 300 # However, some multi-line data blocks can contain arbitrary bytes (for 301 # example, latin-1 or utf-16 data in the body of a message). Commands 302 # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message 303 # data will therefore only accept and produce bytes objects. 304 # Furthermore, since there could be non-compliant servers out there, 305 # we use 'surrogateescape' as the error handler for fault tolerance 306 # and easy round-tripping. This could be useful for some applications 307 # (e.g. NNTP gateways). 308 309 encoding = 'utf-8' 310 errors = 'surrogateescape' 311 312 def __init__(self, host, port=NNTP_PORT, user=None, password=None, 313 readermode=None, usenetrc=False, 314 timeout=_GLOBAL_DEFAULT_TIMEOUT): 315 """Initialize an instance. Arguments: 316 - host: hostname to connect to 317 - port: port to connect to (default the standard NNTP port) 318 - user: username to authenticate with 319 - password: password to use with username 320 - readermode: if true, send 'mode reader' command after 321 connecting. 322 - usenetrc: allow loading username and password from ~/.netrc file 323 if not specified explicitly 324 - timeout: timeout (in seconds) used for socket connections 325 326 readermode is sometimes necessary if you are connecting to an 327 NNTP server on the local machine and intend to call 328 reader-specific commands, such as `group'. If you get 329 unexpected NNTPPermanentErrors, you might need to set 330 readermode. 331 """ 332 self.host = host 333 self.port = port 334 self.sock = self._create_socket(timeout) 335 self.file = None 336 try: 337 self.file = self.sock.makefile("rwb") 338 self._base_init(readermode) 339 if user or usenetrc: 340 self.login(user, password, usenetrc) 341 except: 342 if self.file: 343 self.file.close() 344 self.sock.close() 345 raise 346 347 def _base_init(self, readermode): 348 """Partial initialization for the NNTP protocol. 349 This instance method is extracted for supporting the test code. 350 """ 351 self.debugging = 0 352 self.welcome = self._getresp() 353 354 # Inquire about capabilities (RFC 3977). 355 self._caps = None 356 self.getcapabilities() 357 358 # 'MODE READER' is sometimes necessary to enable 'reader' mode. 359 # However, the order in which 'MODE READER' and 'AUTHINFO' need to 360 # arrive differs between some NNTP servers. If _setreadermode() fails 361 # with an authorization failed error, it will set this to True; 362 # the login() routine will interpret that as a request to try again 363 # after performing its normal function. 364 # Enable only if we're not already in READER mode anyway. 365 self.readermode_afterauth = False 366 if readermode and 'READER' not in self._caps: 367 self._setreadermode() 368 if not self.readermode_afterauth: 369 # Capabilities might have changed after MODE READER 370 self._caps = None 371 self.getcapabilities() 372 373 # RFC 4642 2.2.2: Both the client and the server MUST know if there is 374 # a TLS session active. A client MUST NOT attempt to start a TLS 375 # session if a TLS session is already active. 376 self.tls_on = False 377 378 # Log in and encryption setup order is left to subclasses. 379 self.authenticated = False 380 381 def __enter__(self): 382 return self 383 384 def __exit__(self, *args): 385 is_connected = lambda: hasattr(self, "file") 386 if is_connected(): 387 try: 388 self.quit() 389 except (OSError, EOFError): 390 pass 391 finally: 392 if is_connected(): 393 self._close() 394 395 def _create_socket(self, timeout): 396 if timeout is not None and not timeout: 397 raise ValueError('Non-blocking socket (timeout=0) is not supported') 398 sys.audit("nntplib.connect", self, self.host, self.port) 399 return socket.create_connection((self.host, self.port), timeout) 400 401 def getwelcome(self): 402 """Get the welcome message from the server 403 (this is read and squirreled away by __init__()). 404 If the response code is 200, posting is allowed; 405 if it 201, posting is not allowed.""" 406 407 if self.debugging: print('*welcome*', repr(self.welcome)) 408 return self.welcome 409 410 def getcapabilities(self): 411 """Get the server capabilities, as read by __init__(). 412 If the CAPABILITIES command is not supported, an empty dict is 413 returned.""" 414 if self._caps is None: 415 self.nntp_version = 1 416 self.nntp_implementation = None 417 try: 418 resp, caps = self.capabilities() 419 except (NNTPPermanentError, NNTPTemporaryError): 420 # Server doesn't support capabilities 421 self._caps = {} 422 else: 423 self._caps = caps 424 if 'VERSION' in caps: 425 # The server can advertise several supported versions, 426 # choose the highest. 427 self.nntp_version = max(map(int, caps['VERSION'])) 428 if 'IMPLEMENTATION' in caps: 429 self.nntp_implementation = ' '.join(caps['IMPLEMENTATION']) 430 return self._caps 431 432 def set_debuglevel(self, level): 433 """Set the debugging level. Argument 'level' means: 434 0: no debugging output (default) 435 1: print commands and responses but not body text etc. 436 2: also print raw lines read and sent before stripping CR/LF""" 437 438 self.debugging = level 439 debug = set_debuglevel 440 441 def _putline(self, line): 442 """Internal: send one line to the server, appending CRLF. 443 The `line` must be a bytes-like object.""" 444 sys.audit("nntplib.putline", self, line) 445 line = line + _CRLF 446 if self.debugging > 1: print('*put*', repr(line)) 447 self.file.write(line) 448 self.file.flush() 449 450 def _putcmd(self, line): 451 """Internal: send one command to the server (through _putline()). 452 The `line` must be a unicode string.""" 453 if self.debugging: print('*cmd*', repr(line)) 454 line = line.encode(self.encoding, self.errors) 455 self._putline(line) 456 457 def _getline(self, strip_crlf=True): 458 """Internal: return one line from the server, stripping _CRLF. 459 Raise EOFError if the connection is closed. 460 Returns a bytes object.""" 461 line = self.file.readline(_MAXLINE +1) 462 if len(line) > _MAXLINE: 463 raise NNTPDataError('line too long') 464 if self.debugging > 1: 465 print('*get*', repr(line)) 466 if not line: raise EOFError 467 if strip_crlf: 468 if line[-2:] == _CRLF: 469 line = line[:-2] 470 elif line[-1:] in _CRLF: 471 line = line[:-1] 472 return line 473 474 def _getresp(self): 475 """Internal: get a response from the server. 476 Raise various errors if the response indicates an error. 477 Returns a unicode string.""" 478 resp = self._getline() 479 if self.debugging: print('*resp*', repr(resp)) 480 resp = resp.decode(self.encoding, self.errors) 481 c = resp[:1] 482 if c == '4': 483 raise NNTPTemporaryError(resp) 484 if c == '5': 485 raise NNTPPermanentError(resp) 486 if c not in '123': 487 raise NNTPProtocolError(resp) 488 return resp 489 490 def _getlongresp(self, file=None): 491 """Internal: get a response plus following text from the server. 492 Raise various errors if the response indicates an error. 493 494 Returns a (response, lines) tuple where `response` is a unicode 495 string and `lines` is a list of bytes objects. 496 If `file` is a file-like object, it must be open in binary mode. 497 """ 498 499 openedFile = None 500 try: 501 # If a string was passed then open a file with that name 502 if isinstance(file, (str, bytes)): 503 openedFile = file = open(file, "wb") 504 505 resp = self._getresp() 506 if resp[:3] not in _LONGRESP: 507 raise NNTPReplyError(resp) 508 509 lines = [] 510 if file is not None: 511 # XXX lines = None instead? 512 terminators = (b'.' + _CRLF, b'.\n') 513 while 1: 514 line = self._getline(False) 515 if line in terminators: 516 break 517 if line.startswith(b'..'): 518 line = line[1:] 519 file.write(line) 520 else: 521 terminator = b'.' 522 while 1: 523 line = self._getline() 524 if line == terminator: 525 break 526 if line.startswith(b'..'): 527 line = line[1:] 528 lines.append(line) 529 finally: 530 # If this method created the file, then it must close it 531 if openedFile: 532 openedFile.close() 533 534 return resp, lines 535 536 def _shortcmd(self, line): 537 """Internal: send a command and get the response. 538 Same return value as _getresp().""" 539 self._putcmd(line) 540 return self._getresp() 541 542 def _longcmd(self, line, file=None): 543 """Internal: send a command and get the response plus following text. 544 Same return value as _getlongresp().""" 545 self._putcmd(line) 546 return self._getlongresp(file) 547 548 def _longcmdstring(self, line, file=None): 549 """Internal: send a command and get the response plus following text. 550 Same as _longcmd() and _getlongresp(), except that the returned `lines` 551 are unicode strings rather than bytes objects. 552 """ 553 self._putcmd(line) 554 resp, list = self._getlongresp(file) 555 return resp, [line.decode(self.encoding, self.errors) 556 for line in list] 557 558 def _getoverviewfmt(self): 559 """Internal: get the overview format. Queries the server if not 560 already done, else returns the cached value.""" 561 try: 562 return self._cachedoverviewfmt 563 except AttributeError: 564 pass 565 try: 566 resp, lines = self._longcmdstring("LIST OVERVIEW.FMT") 567 except NNTPPermanentError: 568 # Not supported by server? 569 fmt = _DEFAULT_OVERVIEW_FMT[:] 570 else: 571 fmt = _parse_overview_fmt(lines) 572 self._cachedoverviewfmt = fmt 573 return fmt 574 575 def _grouplist(self, lines): 576 # Parse lines into "group last first flag" 577 return [GroupInfo(*line.split()) for line in lines] 578 579 def capabilities(self): 580 """Process a CAPABILITIES command. Not supported by all servers. 581 Return: 582 - resp: server response if successful 583 - caps: a dictionary mapping capability names to lists of tokens 584 (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] }) 585 """ 586 caps = {} 587 resp, lines = self._longcmdstring("CAPABILITIES") 588 for line in lines: 589 name, *tokens = line.split() 590 caps[name] = tokens 591 return resp, caps 592 593 def newgroups(self, date, *, file=None): 594 """Process a NEWGROUPS command. Arguments: 595 - date: a date or datetime object 596 Return: 597 - resp: server response if successful 598 - list: list of newsgroup names 599 """ 600 if not isinstance(date, (datetime.date, datetime.date)): 601 raise TypeError( 602 "the date parameter must be a date or datetime object, " 603 "not '{:40}'".format(date.__class__.__name__)) 604 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) 605 cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str) 606 resp, lines = self._longcmdstring(cmd, file) 607 return resp, self._grouplist(lines) 608 609 def newnews(self, group, date, *, file=None): 610 """Process a NEWNEWS command. Arguments: 611 - group: group name or '*' 612 - date: a date or datetime object 613 Return: 614 - resp: server response if successful 615 - list: list of message ids 616 """ 617 if not isinstance(date, (datetime.date, datetime.date)): 618 raise TypeError( 619 "the date parameter must be a date or datetime object, " 620 "not '{:40}'".format(date.__class__.__name__)) 621 date_str, time_str = _unparse_datetime(date, self.nntp_version < 2) 622 cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str) 623 return self._longcmdstring(cmd, file) 624 625 def list(self, group_pattern=None, *, file=None): 626 """Process a LIST or LIST ACTIVE command. Arguments: 627 - group_pattern: a pattern indicating which groups to query 628 - file: Filename string or file object to store the result in 629 Returns: 630 - resp: server response if successful 631 - list: list of (group, last, first, flag) (strings) 632 """ 633 if group_pattern is not None: 634 command = 'LIST ACTIVE ' + group_pattern 635 else: 636 command = 'LIST' 637 resp, lines = self._longcmdstring(command, file) 638 return resp, self._grouplist(lines) 639 640 def _getdescriptions(self, group_pattern, return_all): 641 line_pat = re.compile('^(?P<group>[^ \t]+)[ \t]+(.*)$') 642 # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first 643 resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern) 644 if not resp.startswith('215'): 645 # Now the deprecated XGTITLE. This either raises an error 646 # or succeeds with the same output structure as LIST 647 # NEWSGROUPS. 648 resp, lines = self._longcmdstring('XGTITLE ' + group_pattern) 649 groups = {} 650 for raw_line in lines: 651 match = line_pat.search(raw_line.strip()) 652 if match: 653 name, desc = match.group(1, 2) 654 if not return_all: 655 return desc 656 groups[name] = desc 657 if return_all: 658 return resp, groups 659 else: 660 # Nothing found 661 return '' 662 663 def description(self, group): 664 """Get a description for a single group. If more than one 665 group matches ('group' is a pattern), return the first. If no 666 group matches, return an empty string. 667 668 This elides the response code from the server, since it can 669 only be '215' or '285' (for xgtitle) anyway. If the response 670 code is needed, use the 'descriptions' method. 671 672 NOTE: This neither checks for a wildcard in 'group' nor does 673 it check whether the group actually exists.""" 674 return self._getdescriptions(group, False) 675 676 def descriptions(self, group_pattern): 677 """Get descriptions for a range of groups.""" 678 return self._getdescriptions(group_pattern, True) 679 680 def group(self, name): 681 """Process a GROUP command. Argument: 682 - group: the group name 683 Returns: 684 - resp: server response if successful 685 - count: number of articles 686 - first: first article number 687 - last: last article number 688 - name: the group name 689 """ 690 resp = self._shortcmd('GROUP ' + name) 691 if not resp.startswith('211'): 692 raise NNTPReplyError(resp) 693 words = resp.split() 694 count = first = last = 0 695 n = len(words) 696 if n > 1: 697 count = words[1] 698 if n > 2: 699 first = words[2] 700 if n > 3: 701 last = words[3] 702 if n > 4: 703 name = words[4].lower() 704 return resp, int(count), int(first), int(last), name 705 706 def help(self, *, file=None): 707 """Process a HELP command. Argument: 708 - file: Filename string or file object to store the result in 709 Returns: 710 - resp: server response if successful 711 - list: list of strings returned by the server in response to the 712 HELP command 713 """ 714 return self._longcmdstring('HELP', file) 715 716 def _statparse(self, resp): 717 """Internal: parse the response line of a STAT, NEXT, LAST, 718 ARTICLE, HEAD or BODY command.""" 719 if not resp.startswith('22'): 720 raise NNTPReplyError(resp) 721 words = resp.split() 722 art_num = int(words[1]) 723 message_id = words[2] 724 return resp, art_num, message_id 725 726 def _statcmd(self, line): 727 """Internal: process a STAT, NEXT or LAST command.""" 728 resp = self._shortcmd(line) 729 return self._statparse(resp) 730 731 def stat(self, message_spec=None): 732 """Process a STAT command. Argument: 733 - message_spec: article number or message id (if not specified, 734 the current article is selected) 735 Returns: 736 - resp: server response if successful 737 - art_num: the article number 738 - message_id: the message id 739 """ 740 if message_spec: 741 return self._statcmd('STAT {0}'.format(message_spec)) 742 else: 743 return self._statcmd('STAT') 744 745 def next(self): 746 """Process a NEXT command. No arguments. Return as for STAT.""" 747 return self._statcmd('NEXT') 748 749 def last(self): 750 """Process a LAST command. No arguments. Return as for STAT.""" 751 return self._statcmd('LAST') 752 753 def _artcmd(self, line, file=None): 754 """Internal: process a HEAD, BODY or ARTICLE command.""" 755 resp, lines = self._longcmd(line, file) 756 resp, art_num, message_id = self._statparse(resp) 757 return resp, ArticleInfo(art_num, message_id, lines) 758 759 def head(self, message_spec=None, *, file=None): 760 """Process a HEAD command. Argument: 761 - message_spec: article number or message id 762 - file: filename string or file object to store the headers in 763 Returns: 764 - resp: server response if successful 765 - ArticleInfo: (article number, message id, list of header lines) 766 """ 767 if message_spec is not None: 768 cmd = 'HEAD {0}'.format(message_spec) 769 else: 770 cmd = 'HEAD' 771 return self._artcmd(cmd, file) 772 773 def body(self, message_spec=None, *, file=None): 774 """Process a BODY command. Argument: 775 - message_spec: article number or message id 776 - file: filename string or file object to store the body in 777 Returns: 778 - resp: server response if successful 779 - ArticleInfo: (article number, message id, list of body lines) 780 """ 781 if message_spec is not None: 782 cmd = 'BODY {0}'.format(message_spec) 783 else: 784 cmd = 'BODY' 785 return self._artcmd(cmd, file) 786 787 def article(self, message_spec=None, *, file=None): 788 """Process an ARTICLE command. Argument: 789 - message_spec: article number or message id 790 - file: filename string or file object to store the article in 791 Returns: 792 - resp: server response if successful 793 - ArticleInfo: (article number, message id, list of article lines) 794 """ 795 if message_spec is not None: 796 cmd = 'ARTICLE {0}'.format(message_spec) 797 else: 798 cmd = 'ARTICLE' 799 return self._artcmd(cmd, file) 800 801 def slave(self): 802 """Process a SLAVE command. Returns: 803 - resp: server response if successful 804 """ 805 return self._shortcmd('SLAVE') 806 807 def xhdr(self, hdr, str, *, file=None): 808 """Process an XHDR command (optional server extension). Arguments: 809 - hdr: the header type (e.g. 'subject') 810 - str: an article nr, a message id, or a range nr1-nr2 811 - file: Filename string or file object to store the result in 812 Returns: 813 - resp: server response if successful 814 - list: list of (nr, value) strings 815 """ 816 pat = re.compile('^([0-9]+) ?(.*)\n?') 817 resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file) 818 def remove_number(line): 819 m = pat.match(line) 820 return m.group(1, 2) if m else line 821 return resp, [remove_number(line) for line in lines] 822 823 def xover(self, start, end, *, file=None): 824 """Process an XOVER command (optional server extension) Arguments: 825 - start: start of range 826 - end: end of range 827 - file: Filename string or file object to store the result in 828 Returns: 829 - resp: server response if successful 830 - list: list of dicts containing the response fields 831 """ 832 resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end), 833 file) 834 fmt = self._getoverviewfmt() 835 return resp, _parse_overview(lines, fmt) 836 837 def over(self, message_spec, *, file=None): 838 """Process an OVER command. If the command isn't supported, fall 839 back to XOVER. Arguments: 840 - message_spec: 841 - either a message id, indicating the article to fetch 842 information about 843 - or a (start, end) tuple, indicating a range of article numbers; 844 if end is None, information up to the newest message will be 845 retrieved 846 - or None, indicating the current article number must be used 847 - file: Filename string or file object to store the result in 848 Returns: 849 - resp: server response if successful 850 - list: list of dicts containing the response fields 851 852 NOTE: the "message id" form isn't supported by XOVER 853 """ 854 cmd = 'OVER' if 'OVER' in self._caps else 'XOVER' 855 if isinstance(message_spec, (tuple, list)): 856 start, end = message_spec 857 cmd += ' {0}-{1}'.format(start, end or '') 858 elif message_spec is not None: 859 cmd = cmd + ' ' + message_spec 860 resp, lines = self._longcmdstring(cmd, file) 861 fmt = self._getoverviewfmt() 862 return resp, _parse_overview(lines, fmt) 863 864 def date(self): 865 """Process the DATE command. 866 Returns: 867 - resp: server response if successful 868 - date: datetime object 869 """ 870 resp = self._shortcmd("DATE") 871 if not resp.startswith('111'): 872 raise NNTPReplyError(resp) 873 elem = resp.split() 874 if len(elem) != 2: 875 raise NNTPDataError(resp) 876 date = elem[1] 877 if len(date) != 14: 878 raise NNTPDataError(resp) 879 return resp, _parse_datetime(date, None) 880 881 def _post(self, command, f): 882 resp = self._shortcmd(command) 883 # Raises a specific exception if posting is not allowed 884 if not resp.startswith('3'): 885 raise NNTPReplyError(resp) 886 if isinstance(f, (bytes, bytearray)): 887 f = f.splitlines() 888 # We don't use _putline() because: 889 # - we don't want additional CRLF if the file or iterable is already 890 # in the right format 891 # - we don't want a spurious flush() after each line is written 892 for line in f: 893 if not line.endswith(_CRLF): 894 line = line.rstrip(b"\r\n") + _CRLF 895 if line.startswith(b'.'): 896 line = b'.' + line 897 self.file.write(line) 898 self.file.write(b".\r\n") 899 self.file.flush() 900 return self._getresp() 901 902 def post(self, data): 903 """Process a POST command. Arguments: 904 - data: bytes object, iterable or file containing the article 905 Returns: 906 - resp: server response if successful""" 907 return self._post('POST', data) 908 909 def ihave(self, message_id, data): 910 """Process an IHAVE command. Arguments: 911 - message_id: message-id of the article 912 - data: file containing the article 913 Returns: 914 - resp: server response if successful 915 Note that if the server refuses the article an exception is raised.""" 916 return self._post('IHAVE {0}'.format(message_id), data) 917 918 def _close(self): 919 try: 920 if self.file: 921 self.file.close() 922 del self.file 923 finally: 924 self.sock.close() 925 926 def quit(self): 927 """Process a QUIT command and close the socket. Returns: 928 - resp: server response if successful""" 929 try: 930 resp = self._shortcmd('QUIT') 931 finally: 932 self._close() 933 return resp 934 935 def login(self, user=None, password=None, usenetrc=True): 936 if self.authenticated: 937 raise ValueError("Already logged in.") 938 if not user and not usenetrc: 939 raise ValueError( 940 "At least one of `user` and `usenetrc` must be specified") 941 # If no login/password was specified but netrc was requested, 942 # try to get them from ~/.netrc 943 # Presume that if .netrc has an entry, NNRP authentication is required. 944 try: 945 if usenetrc and not user: 946 import netrc 947 credentials = netrc.netrc() 948 auth = credentials.authenticators(self.host) 949 if auth: 950 user = auth[0] 951 password = auth[2] 952 except OSError: 953 pass 954 # Perform NNTP authentication if needed. 955 if not user: 956 return 957 resp = self._shortcmd('authinfo user ' + user) 958 if resp.startswith('381'): 959 if not password: 960 raise NNTPReplyError(resp) 961 else: 962 resp = self._shortcmd('authinfo pass ' + password) 963 if not resp.startswith('281'): 964 raise NNTPPermanentError(resp) 965 # Capabilities might have changed after login 966 self._caps = None 967 self.getcapabilities() 968 # Attempt to send mode reader if it was requested after login. 969 # Only do so if we're not in reader mode already. 970 if self.readermode_afterauth and 'READER' not in self._caps: 971 self._setreadermode() 972 # Capabilities might have changed after MODE READER 973 self._caps = None 974 self.getcapabilities() 975 976 def _setreadermode(self): 977 try: 978 self.welcome = self._shortcmd('mode reader') 979 except NNTPPermanentError: 980 # Error 5xx, probably 'not implemented' 981 pass 982 except NNTPTemporaryError as e: 983 if e.response.startswith('480'): 984 # Need authorization before 'mode reader' 985 self.readermode_afterauth = True 986 else: 987 raise 988 989 if _have_ssl: 990 def starttls(self, context=None): 991 """Process a STARTTLS command. Arguments: 992 - context: SSL context to use for the encrypted connection 993 """ 994 # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if 995 # a TLS session already exists. 996 if self.tls_on: 997 raise ValueError("TLS is already enabled.") 998 if self.authenticated: 999 raise ValueError("TLS cannot be started after authentication.") 1000 resp = self._shortcmd('STARTTLS') 1001 if resp.startswith('382'): 1002 self.file.close() 1003 self.sock = _encrypt_on(self.sock, context, self.host) 1004 self.file = self.sock.makefile("rwb") 1005 self.tls_on = True 1006 # Capabilities may change after TLS starts up, so ask for them 1007 # again. 1008 self._caps = None 1009 self.getcapabilities() 1010 else: 1011 raise NNTPError("TLS failed to start.") 1012 1013 1014if _have_ssl: 1015 class NNTP_SSL(NNTP): 1016 1017 def __init__(self, host, port=NNTP_SSL_PORT, 1018 user=None, password=None, ssl_context=None, 1019 readermode=None, usenetrc=False, 1020 timeout=_GLOBAL_DEFAULT_TIMEOUT): 1021 """This works identically to NNTP.__init__, except for the change 1022 in default port and the `ssl_context` argument for SSL connections. 1023 """ 1024 self.ssl_context = ssl_context 1025 super().__init__(host, port, user, password, readermode, 1026 usenetrc, timeout) 1027 1028 def _create_socket(self, timeout): 1029 sock = super()._create_socket(timeout) 1030 try: 1031 sock = _encrypt_on(sock, self.ssl_context, self.host) 1032 except: 1033 sock.close() 1034 raise 1035 else: 1036 return sock 1037 1038 __all__.append("NNTP_SSL") 1039 1040 1041# Test retrieval when run as a script. 1042if __name__ == '__main__': 1043 import argparse 1044 1045 parser = argparse.ArgumentParser(description="""\ 1046 nntplib built-in demo - display the latest articles in a newsgroup""") 1047 parser.add_argument('-g', '--group', default='gmane.comp.python.general', 1048 help='group to fetch messages from (default: %(default)s)') 1049 parser.add_argument('-s', '--server', default='news.gmane.io', 1050 help='NNTP server hostname (default: %(default)s)') 1051 parser.add_argument('-p', '--port', default=-1, type=int, 1052 help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT)) 1053 parser.add_argument('-n', '--nb-articles', default=10, type=int, 1054 help='number of articles to fetch (default: %(default)s)') 1055 parser.add_argument('-S', '--ssl', action='store_true', default=False, 1056 help='use NNTP over SSL') 1057 args = parser.parse_args() 1058 1059 port = args.port 1060 if not args.ssl: 1061 if port == -1: 1062 port = NNTP_PORT 1063 s = NNTP(host=args.server, port=port) 1064 else: 1065 if port == -1: 1066 port = NNTP_SSL_PORT 1067 s = NNTP_SSL(host=args.server, port=port) 1068 1069 caps = s.getcapabilities() 1070 if 'STARTTLS' in caps: 1071 s.starttls() 1072 resp, count, first, last, name = s.group(args.group) 1073 print('Group', name, 'has', count, 'articles, range', first, 'to', last) 1074 1075 def cut(s, lim): 1076 if len(s) > lim: 1077 s = s[:lim - 4] + "..." 1078 return s 1079 1080 first = str(int(last) - args.nb_articles + 1) 1081 resp, overviews = s.xover(first, last) 1082 for artnum, over in overviews: 1083 author = decode_header(over['from']).split('<', 1)[0] 1084 subject = decode_header(over['subject']) 1085 lines = int(over[':lines']) 1086 print("{:7} {:20} {:42} ({})".format( 1087 artnum, cut(author, 20), cut(subject, 42), lines) 1088 ) 1089 1090 s.quit() 1091