diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf4db41092..0df9b8fb64 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,10 @@ Next release ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 +- Fix a regression in PDF text extraction that silently stopped after the + first page, so copyright/license detection missed text on later pages. + https://github.com/aboutcode-org/scancode-toolkit/pull/5260 + - Improve copyright detection for statements with parens or trailing "authors" diff --git a/src/textcode/pdf.py b/src/textcode/pdf.py index 4e90c866f8..1c966a3a66 100644 --- a/src/textcode/pdf.py +++ b/src/textcode/pdf.py @@ -43,5 +43,5 @@ def get_text_lines(location, max_pages=5): interpreter.process_page(page) if max_pages and page_num == max_pages: break - extracted_text.seek(0) - return extracted_text.readlines() + extracted_text.seek(0) + return extracted_text.readlines() diff --git a/tests/textcode/data/pdf/multi_page.pdf b/tests/textcode/data/pdf/multi_page.pdf new file mode 100644 index 0000000000..d369663e18 Binary files /dev/null and b/tests/textcode/data/pdf/multi_page.pdf differ diff --git a/tests/textcode/test_pdf.py b/tests/textcode/test_pdf.py index 7941953568..cc7b2d05da 100644 --- a/tests/textcode/test_pdf.py +++ b/tests/textcode/test_pdf.py @@ -51,6 +51,33 @@ def get_text(location): assert result == expected + def test_get_text_lines_extracts_all_pages_up_to_max_pages(self): + # regression test: text extraction must not stop after the first page + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file) + text = b''.join(result) + assert b'This is page 1 of a multi-page test document.' in text + assert b'This notice is on page 2.' in text + assert b'The last page 5 of this test document.' in text + # the default max_pages=5 must still be honored + assert b'Page 6' not in text + + def test_get_text_lines_returns_lines_when_max_pages_is_one(self): + # regression test: reaching max_pages must not return None + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file, max_pages=1) + assert result + text = b''.join(result) + assert b'This is page 1 of a multi-page test document.' in text + assert b'page 2' not in text + + def test_get_text_lines_extracts_all_pages_when_max_pages_is_zero(self): + test_file = self.get_test_loc('pdf/multi_page.pdf') + result = pdf.get_text_lines(test_file, max_pages=0) + text = b''.join(result) + assert b'Page 6 is beyond the default max_pages limit.' in text + assert b'Page 7 is also beyond the default max_pages limit.' in text + def test_pdfminer_can_parse_faulty_broadcom_doc(self): # test for https://github.com/euske/pdfminer/issues/118 test_file = self.get_test_loc('pdf/pdfminer_bug_118/faulty.pdf')