From 6978ab0563b1b266ca087eb31ac5144de453f445 Mon Sep 17 00:00:00 2001 From: Michael Brandt Date: Sun, 17 Nov 2024 11:32:59 -0700 Subject: [PATCH] Add pagination on collections with test --- .../beethoven_collection_urls.txt | 223 + .../beethoven_collections_view.html | 6619 ----------------- .../tests/test_imslp_collection_scraper.py | 13 +- data_ingest/tests/test_parse_movements.py | 2 - data_ingest/utils/__init__.py | 4 +- data_ingest/utils/imslp_scraping.py | 144 +- 6 files changed, 317 insertions(+), 6688 deletions(-) create mode 100644 data_ingest/tests/scrape_responses/beethoven_collection_urls.txt delete mode 100644 data_ingest/tests/scrape_responses/beethoven_collections_view.html diff --git a/data_ingest/tests/scrape_responses/beethoven_collection_urls.txt b/data_ingest/tests/scrape_responses/beethoven_collection_urls.txt new file mode 100644 index 0000000..b5c0ad8 --- /dev/null +++ b/data_ingest/tests/scrape_responses/beethoven_collection_urls.txt @@ -0,0 +1,223 @@ +https://imslp.org/wiki/Musikalischer_Hausschatz_der_Deutschen_(Fink%2C_Gottfried_Wilhelm) +https://imslp.org/wiki/6_Deutsche_Lieder_(Various) +https://imslp.org/wiki/Beethoven's_Masterpieces_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Harmonium-Album_(Blumenthal%2C_Paul) +https://imslp.org/wiki/Die_Wundergeige_(Seybold%2C_Arthur) +https://imslp.org/wiki/Symphonien_f%C3%BCr_Pianoforte_und_Violine_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/51_Gemischte_Ch%C3%B6re_(Lafite%2C_Carl) +https://imslp.org/wiki/Useful_Voluntaries_(Pearce%2C_Charles_William) +https://imslp.org/wiki/24_Morceaux_choisis_dans_les_quatuors_et_quintettes_(Cramer%2C_Johann_Baptist) +https://imslp.org/wiki/Klassische_Sinfonien_und_Kammermusik_f%C3%BCr_Piano_und_Violine_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Sacred_Minstrelsy_(Parker%2C_John_William) +https://imslp.org/wiki/Sonatiny_(Micha%C5%82owski%2C_Aleksander) +https://imslp.org/wiki/L'%C3%A9cole_des_grands_ma%C3%AEtres%2C_Op.119_(Barbot%2C_Paul) +https://imslp.org/wiki/St%C3%BCcke_alter_Meister_(Hollaender%2C_Gustav) +https://imslp.org/wiki/Liederhort_(Riemann%2C_Hugo) +https://imslp.org/wiki/Un_Quaderno_di_Autografi_di_Beethoven_del_1825_(Roda%2C_Cecilio_de) +https://imslp.org/wiki/18_Deutsche_Gedichte_(Various) +https://imslp.org/wiki/St%C3%BCcke_alter_Meister_(Burmester%2C_Willy) +https://imslp.org/wiki/Bunte_B%C3%BChne_(Batka%2C_Richard) +https://imslp.org/wiki/Ouverturen-Album_(Ulrich%2C_Hugo) +https://imslp.org/wiki/The_Hundred_Best_Short_Classics_(Whitemore%2C_Cuthbert_F.) +https://imslp.org/wiki/German%2C_French_and_Italian_Song_Classics_(Parker%2C_Horatio) +https://imslp.org/wiki/Sonate_per_pianoforte_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Gesamtausgabe_der_Werke_Ludwig_van_Beethovens_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Trios_faciles_(Wilhelm_Hansen) +https://imslp.org/wiki/Ornamentik_in_Beethovens_Klavierwerken_(Ehrlich%2C_Heinrich) +https://imslp.org/wiki/Sketchbooks_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Deutsche_Weisen_(Various) +https://imslp.org/wiki/Sonata_Album_(G._Schirmer) +https://imslp.org/wiki/Sonaten_f%C3%BCr_Pianoforte_und_Violine_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/17_Streichquartette_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Praktyczna_Szko%C5%82a_Gry_na_Mandolinie_(Plohn%2C_Alfred) +https://imslp.org/wiki/Our_Favorite_Tunes_(Hermann%2C_Friedrich) +https://imslp.org/wiki/The_Ditson_Easy_Trio_Album_(Rissland%2C_Karl) +https://imslp.org/wiki/9_Tonst%C3%BCcke_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Le_concert_au_salon_(Vogel%2C_Charles) +https://imslp.org/wiki/Touch_and_Technic%2C_Op.44_(Mason%2C_William) +https://imslp.org/wiki/Duos%2C_Trios%2C_Quartette%2C_Quintette%2C_Sextette_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Opern-Perlen_(Various) +https://imslp.org/wiki/2_Cello_Sonatas%2C_Op.5_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/The_Organist%E2%80%99s_Journal_(Archer%2C_Frederic) +https://imslp.org/wiki/Sammlung_klassischer_und_moderner_St%C3%BCcke_(Sitt%2C_Hans) +https://imslp.org/wiki/A_Select_Collection_of_Original_Irish_Airs_(Thomson%2C_George) +https://imslp.org/wiki/Gedichte_von_Goethe_in_Compositionen_(Friedlaender%2C_Max) +https://imslp.org/wiki/Alte_Meister_f%C3%BCr_Junge_Spieler_(Moffat%2C_Alfred) +https://imslp.org/wiki/The_Village_Organist_(Matthews%2C_Timothy_Richard) +https://imslp.org/wiki/12_Sonates_de_Haydn%2C_Mozart_et_Beethoven_(Delisse%2C_Paul) +https://imslp.org/wiki/Mittell's_Violin_Classics_(Mittell%2C_Philipp) +https://imslp.org/wiki/Trio-Albums_(Various) +https://imslp.org/wiki/Ouvertures_pour_piano_et_violon_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Imperial_Edition_Contralto_Songs_(Various) +https://imslp.org/wiki/Album_for_Violin_and_Harmonium_(Hansen%2C_Nikolaj) +https://imslp.org/wiki/Instructive_Album_(Foote%2C_Arthur) +https://imslp.org/wiki/A_Scrap_Book_for_the_Use_of_Students_of_the_Double_Bass_(Reynolds%2C_John) +https://imslp.org/wiki/%C3%89chos_du_monde_religieux_(Various) +https://imslp.org/wiki/Harmonium-Album_(Various) +https://imslp.org/wiki/15_Songs_by_Beethoven_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Antologia_Pianistica_(Piccioli%2C_Giuseppe) +https://imslp.org/wiki/Piano_Sonatas_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Ouvert%C3%BCren_f%C3%BCr_2_Pianoforte_zu_8_H%C3%A4nden_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Original_Compositions_and_Adaptations_for_the_Harp_(Robinson%2C_Gertrude_Ina) +https://imslp.org/wiki/School_of_Double_Notes_(Philipp%2C_Isidor) +https://imslp.org/wiki/Useful_Teaching_Songs_for_All_Voices_(Lehmann%2C_Liza) +https://imslp.org/wiki/56_Songs_You_Like_to_Sing_(Various) +https://imslp.org/wiki/Studio_generale_(Czerny%2C_Carl) +https://imslp.org/wiki/Trios_und_Serenade_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Wiener_Pfennig-Magazin_(Czerny%2C_Carl) +https://imslp.org/wiki/Oeuvres_compl%C3%A8tes_pour_piano_seul_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Violin_Solos_(Andr%C3%A9_Jr.%2C_Anton) +https://imslp.org/wiki/Cadenzas_for_Piano_Concertos_Nos.1-4_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Le_guide_du_lecteur%2C_Op.281_(Duvernoy%2C_Jean-Baptiste) +https://imslp.org/wiki/5_Canons_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/New_Imperial_Edition_Contralto_Songs_(Northcote%2C_Sydney) +https://imslp.org/wiki/Gray_Album_of_20_Pieces_for_the_Organ_(Various) +https://imslp.org/wiki/In_questa_tomba_oscura_(Various) +https://imslp.org/wiki/Sonatinen-Album_(K%C3%B6hler%2C_Louis) +https://imslp.org/wiki/M%C3%A4rsche_f%C3%BCr_zwei_Pianoforte_zu_acht_H%C3%A4nden_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Concert_Violin_Solos_the_Whole_World_Plays_(Wier%2C_Albert_Ernest) +https://imslp.org/wiki/Airs_populaires%2C_chants_nationaux_et_motifs_c%C3%A9l%C3%A8bres_(Vilbac%2C_Renaud_de) +https://imslp.org/wiki/Lady's_Musical_Library_(Jarvis%2C_Charles) +https://imslp.org/wiki/Ludwig_van_Beethoven_(Frimmel%2C_Theodor_von) +https://imslp.org/wiki/A_March_Album_for_the_Organ_(Various) +https://imslp.org/wiki/Dance_Movements_from_the_Works_of_Great_Masters_(Hermann%2C_Friedrich) +https://imslp.org/wiki/3_String_Trios%2C_Op.9_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Piano_Compositions_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Variationen_f%C3%BCr_Piano_und_Violoncell_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/L'harmonium-concertant_(Vilbac%2C_Renaud_de) +https://imslp.org/wiki/Master_Series_For_The_Young_(Hughes%2C_Edwin) +https://imslp.org/wiki/Symphonien_f%C3%BCr_Piano_Solo_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Beethoven-Album_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Album_of_Symphony_Themes_for_the_Junior_String_Orchestra_(Various) +https://imslp.org/wiki/Album_of_Transcriptions_for_Piano_Four-hands_(Oesterle%2C_Louis) +https://imslp.org/wiki/38_Piano_Sonatas_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Die_Schule_des_Octavenspiels%2C_Op.48_(Kullak%2C_Theodor) +https://imslp.org/wiki/Streich-Quartett-Album_(Zanger%2C_Gustav) +https://imslp.org/wiki/Symphonien_f%C3%BCr_Pianoforte_zu_4_H%C3%A4nden_mit_Violine_und_Violoncello_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/12_Duettini_(Various) +https://imslp.org/wiki/Symphonies_pour_Piano_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/La_virtuosit%C3%A9%2C_Op.44_(Dumont%2C_F%C3%A9lix) +https://imslp.org/wiki/Variationen_f%C3%BCr_Pianoforte_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Guitar_Album%2C_Op.27_(Dorn%2C_Charles_James) +https://imslp.org/wiki/Les_quatre_pianistes_(Schultze%2C_Max) +https://imslp.org/wiki/S%C3%A4mtliche_Streichquartette_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/L'Harmonium-Concertant_(Lange%2C_Richard) +https://imslp.org/wiki/Imperial_Edition_Tenor_Songs_(Various) +https://imslp.org/wiki/Kadenzen_zu_klassischen_Klavierkonzerten%2C_Op.69_(Schumann%2C_Georg) +https://imslp.org/wiki/Instructions_for_the_Piano_Forte_(Cramer%2C_Johann_Baptist) +https://imslp.org/wiki/Meister_f%C3%BCr_die_Jugend_(Various) +https://imslp.org/wiki/Kleine_und_leichte_%C3%9Cbungsst%C3%BCcke_im_Klavierspielen_(Various) +https://imslp.org/wiki/Baroque_and_Classical_Pieces_for_the_Piano_(Herring%2C_Francis) +https://imslp.org/wiki/Les_trios_symphoniques_(Vilbac%2C_Renaud_de) +https://imslp.org/wiki/Sammlung_ber%C3%BChmter_Compositionen_(Salter%2C_Norbert) +https://imslp.org/wiki/Quartett-S%C3%A4tze_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Klavier-Werke_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Children's_Piano_Pieces_the_Whole_World_Plays_(Wier%2C_Albert_Ernest) +https://imslp.org/wiki/Le_mus%C3%A9e_des_pianistes_(Various) +https://imslp.org/wiki/Pianoforte-Album_(Various) +https://imslp.org/wiki/Polyhymnia%2C_Op.40_(Reinhard%2C_August) +https://imslp.org/wiki/New_Imperial_Edition_Tenor_Songs_(Northcote%2C_Sydney) +https://imslp.org/wiki/Practische_Pianoforteschule_(Various) +https://imslp.org/wiki/Lieder_und_Ges%C3%A4nge%2C_D-Tu_Mk_46-1_(Various) +https://imslp.org/wiki/101_Classics_(Dunstan%2C_Ralph) +https://imslp.org/wiki/Bone_Mass_Collection_(Reifsnyder%2C_Bob) +https://imslp.org/wiki/10_Popular_Pieces_of_Moderate_Difficulty_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Il_mio_primo_Beethoven_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/New_Imperial_Edition_Mezzo-Soprano_Songs_(Northcote%2C_Sydney) +https://imslp.org/wiki/Gar%C5%9B%C4%87_pie%C5%9Bni_(Bursa%2C_Stanis%C5%82aw) +https://imslp.org/wiki/Classic_Keyboard_Music_(Sauer%2C_Emil_von) +https://imslp.org/wiki/Sonntags-musik_(Pauer%2C_Ernst) +https://imslp.org/wiki/Souvenirs_de_musique_de_chambre_(Alkan%2C_Charles-Valentin) +https://imslp.org/wiki/Re-discovered_Classics_for_Piano_(Henderson%2C_Archibald_Martin) +https://imslp.org/wiki/Sacred_Songs%2C_Ancient_and_Modern_(Hiles%2C_John) +https://imslp.org/wiki/Album_of_Classical_Themes_for_the_Junior_String_Orchestra_(Various) +https://imslp.org/wiki/3_String_Quartets%2C_Op.59_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Ber%C3%BChmte_Ouverturen_von_Mozart%2C_Beethoven%2C_Weber%2C_Mendelssohn%2C_Bellini%2C_Rossini_(Various) +https://imslp.org/wiki/Konzerte_und_Phantasie_f%C3%BCr_Klavier_zu_zwei_H%C3%A4nden_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Practical_Tutor_for_the_Double-Bass_(Langey%2C_Otto) +https://imslp.org/wiki/Liederbuch%2C_D-Tu_Mk_45_(Various) +https://imslp.org/wiki/Ludwig_van_Beethovens_Werke_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Blumenlese_f%C3%BCr_Klavierliebhaber_(Bossler%2C_Heinrich_Philipp) +https://imslp.org/wiki/Musikalisches_Angebinde_zum_Neuen_Jahre_(M%C3%BCller%2C_Carl_Friedrich) +https://imslp.org/wiki/40_Transcriptions_(Oesterle%2C_Louis) +https://imslp.org/wiki/Sonaten_f%C3%BCr_Klavier_zu_Zwei_H%C3%A4nden_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Complete_Piano_Sonatas_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Sammlung_klassischer_St%C3%BCcke_aus_Werken_ber%C3%BChmter_Meister_(Various) +https://imslp.org/wiki/Biblioteca_d'oro_(Longo%2C_Alessandro) +https://imslp.org/wiki/Introduction_to_the_Art_of_Playing_the_Pianoforte%2C_Op.42_(Clementi%2C_Muzio) +https://imslp.org/wiki/12_Waltzes_by_Beethoven_(Callcott%2C_William_Hutchins) +https://imslp.org/wiki/Album_of_Classical_Sonatinas_(Seitz%2C_Friedrich) +https://imslp.org/wiki/Beauties_of_Sacred_Harmony_(Nightingale%2C_John_Charles) +https://imslp.org/wiki/Klassisches_Vortrags-Album_(Klengel%2C_Paul) +https://imslp.org/wiki/Ballet-Album_(Kleinmichel%2C_Richard) +https://imslp.org/wiki/S%C3%A4mmtliche_Sonaten_fur_Pianoforte_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/The_Songs_of_England_(Hatton%2C_John_Liptrot) +https://imslp.org/wiki/Transcriptions_of_Beethoven's_Quartets_(Chausson%2C_Ernest) +https://imslp.org/wiki/15_Fugues_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/S%C3%A4mmtliche_Compositionen_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Trifolien%2C_Op.33_(Streben%2C_Ernst) +https://imslp.org/wiki/40_%C3%89tudes_r%C3%A9cr%C3%A9atives%2C_Op.39_(Guichard%2C_A.) +https://imslp.org/wiki/T%C3%A4nze_f%C3%BCr_Orchester_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/The_Kneisel_Collection_(Kneisel%2C_Franz) +https://imslp.org/wiki/Instruktive_G%C3%A4nge_durch_die_Kompositionen_von_Haydn%2C_Mozart_und_Beethoven_(Eschmann%2C_Johann_Carl) +https://imslp.org/wiki/Masterpieces_of_Piano_Music_(Wier%2C_Albert_Ernest) +https://imslp.org/wiki/Les_grands_ma%C3%AEtres_de_l'art_musical_(Z%C3%B6llner%2C_Heinrich) +https://imslp.org/wiki/Ouverturen_f%C3%BCr_Piano_und_Violine_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Le_Petit_Pianiste_(Philipp%2C_Isidor) +https://imslp.org/wiki/Beethoven_intimo_(Casella%2C_Alfredo) +https://imslp.org/wiki/60_Soft_Voluntaries_(Westrop%2C_Thomas) +https://imslp.org/wiki/The_School_and_Community_Orchestra_(Gordon%2C_Louis_Morton) +https://imslp.org/wiki/Harmonium-Album_(Stapf%2C_Ernst) +https://imslp.org/wiki/6_Piano_Sonatinas_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Adagios_de_Beethoven%2C_Op.101_(Brisson%2C_Fr%C3%A9d%C3%A9ric) +https://imslp.org/wiki/Trios_f%C3%BCr_Streichinstrumente_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Association_Hymn_Book_(YMCA) +https://imslp.org/wiki/Concert_und_Romanzen_f%C3%BCr_Violine_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/75_Eight-bar_Studies_in_Pianoforte_Technique_(Denn%C3%A9e%2C_Charles) +https://imslp.org/wiki/Concentus_Sacri_(Rosewig%2C_Albert_Henry) +https://imslp.org/wiki/La_virtuosit%C3%A9_(Le_Couppey%2C_F%C3%A9lix) +https://imslp.org/wiki/Concerte_f%C3%BCr_Pianoforte_zu_vier_H%C3%A4nden_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Music_Performed_at_Wellington's_Funeral_(Goss%2C_John) +https://imslp.org/wiki/Blue_Album_of_20_Pieces_for_the_Organ_(Various) +https://imslp.org/wiki/Trios_f%C3%BCr_Pianoforte%2C_Violine_und_Violoncell_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Anthology_of_German_Piano_Music_(Moszkowski%2C_Moritz) +https://imslp.org/wiki/Sang_und_Klang_im_19._und_20._Jahrhundert_(Various) +https://imslp.org/wiki/A_Collection_of_German_Overtures_(Oesterle%2C_Louis) +https://imslp.org/wiki/Concerte_f%C3%BCr_das_Pianoforte_allein_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Standard_Graded_Course_of_Studies_for_the_Pianoforte_(Mathews%2C_William_Smythe_Babcock) +https://imslp.org/wiki/Symphonien_f%C3%BCr_Pianoforte_zu_vier_H%C3%A4nden_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Oesterle's_Instructive_Course_for_the_Piano_(Oesterle%2C_Louis) +https://imslp.org/wiki/Du_himmlisch_unbekanntes_Wesen_(Griepenkerl%2C_Friedrich_Conrad) +https://imslp.org/wiki/15_M%C3%A9lodies_et_airs_populaires%2C_Op.17_(L%C3%A9onard_de_La_Tuilerie%2C_Auguste) +https://imslp.org/wiki/Novello's_Part-Song_Book_(Various) +https://imslp.org/wiki/Pi%C3%A8ces_choisies_faciles_pour_le_Pianoforte_(Various) +https://imslp.org/wiki/Alte_T%C3%A4nze_(Pauer%2C_Ernst) +https://imslp.org/wiki/S%C3%A4mtliche_Sonaten_f%C3%BCr_Klavier_zu_zwei_H%C3%A4nden_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/6_String_Quartets%2C_Op.18_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Bunte_Bl%C3%A4tter_(Karg-Elert%2C_Sigfrid) +https://imslp.org/wiki/Gems_of_Antiquity_(Neitzel%2C_Otto) +https://imslp.org/wiki/Violin-Terzette_(Gr%C3%BCnwald%2C_Adolf) +https://imslp.org/wiki/3_Piano_Sonatas%2C_WoO_47_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Piano_Transcriptions_of_Various_Composers_(Friedman%2C_Ignaz) +https://imslp.org/wiki/Ungdommens_Melodi-Album_(Hansen%2C_Nikolaj) +https://imslp.org/wiki/Quartette_von_Haydn%2C_Mozart%2C_Beethoven_f%C3%BCr_Klavier_und_Violine_(Hermann%2C_Friedrich) +https://imslp.org/wiki/Violin_Pieces_the_Whole_World_Plays_(Wier%2C_Albert_Ernest) +https://imslp.org/wiki/Trios%2C_Quartette_und_Quintette_f%C3%BCr_Streichinstrumente_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Souvenirs_des_concerts_du_Conservatoire_(Alkan%2C_Charles-Valentin) +https://imslp.org/wiki/Vortrags-Album_(Klengel%2C_Paul) +https://imslp.org/wiki/Quintette_f%C3%BCr_2_Violinen%2C_2_Bratschen_und_Violoncell_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Quartett-Album_(Sitt%2C_Hans) +https://imslp.org/wiki/S%C3%A4mtliche_Ouverturen_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Ausgew%C3%A4hlte_Lieder_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/Musikaliskt_Tidsf%C3%B6rdrif_(Various) +https://imslp.org/wiki/The_Sacred_Musical_Offering_(Purday%2C_Charles_Henry) +https://imslp.org/wiki/Ach%2C_Du_gehst_ohne_mich_ins_Heimatland!_(Griepenkerl%2C_Friedrich_Conrad) +https://imslp.org/wiki/Alte_Weisen_(Burmester%2C_Willy) +https://imslp.org/wiki/Neujahrs_und_Carnevalsgabe_'Seyd_uns_zum_zweytenmal_willkommen'_(M%C3%BCller%2C_Carl_Friedrich) +https://imslp.org/wiki/Posaunen-Trios_(Bamberg%2C_Karl) +https://imslp.org/wiki/Album_of_Bass_Songs_(Various) +https://imslp.org/wiki/Le_tr%C3%A9sor_des_pianistes_(Farrenc%2C_Aristide) +https://imslp.org/wiki/Brown_Album_of_20_Pieces_for_the_Organ_(Various) +https://imslp.org/wiki/3_Dances_(Beethoven%2C_Ludwig_van) +https://imslp.org/wiki/The_Amateur's_Musical_Library_(Jarvis%2C_Charles) +https://imslp.org/wiki/Klaviersonaten_(Beethoven%2C_Ludwig_van) diff --git a/data_ingest/tests/scrape_responses/beethoven_collections_view.html b/data_ingest/tests/scrape_responses/beethoven_collections_view.html deleted file mode 100644 index 48fbbd3..0000000 --- a/data_ingest/tests/scrape_responses/beethoven_collections_view.html +++ /dev/null @@ -1,6619 +0,0 @@ - - - - Category:Beethoven, Ludwig van - IMSLP - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - -
-
-
-
-
- - -

- Category:Beethoven, Ludwig van -

-
-
-
-
-
-
- - Ludwig van Beethoven - -
-
- - Signature - -
-
-
-

- - Ludwig van Beethoven - -

- (16 December 1770 — 26 March 1827) -
-
-
- ＝ -
- - Alternative Names/Transliterations: Louis van Beethoven - -
-
- ＝ -
- - Name in Other Languages: - - Ludovicus van Beethoven - - , - - Лиудвиг ван Бетховен - - , - - ሉድቪግ ቫን ቤትሆቨን - - , - - Hloðwig van Beethoven - - , - - لودفيج فان بيتهوفن - - , - - [ - - 57 more... - - ] - - - - ܠܘܕܒܝܓ ܒܐܢ ܒܝܛܗܘܒܢ - - , - - بيتهوفين - - , - - Lüdviq van Bethoven - - , - - Людвиг ван Бетховен - - , - - Ludwig Van Beethoven - - , - - Людвіг ван Бетховен - - , - - Лудвиг ван Бетховен - - , - - লুডভিগ ফান বেটহোফেন - - , - - لودڤیگ ڤان بێتھۆڤن - - , - - Λούντβιχ βαν Μπετόβεν - - , - - لودویگ فان بتهوون - - , - - Lüdvig van Bethoven - - , - - 貝多芬 - - , - - Pi-tô-fûn - - , - - לודוויג ואן בטהובן - - , - - Լյուդվիգ վան Բեթհովեն - - , - - ルートヴィヒ・ヴァン・ベートーヴェン - - , - - ludvix.fan.beitoven - - , - - ლუდვიგ ვან ბეთჰოვენი - - , - - ಲುಡ್ವಿಗ್ ವಾನ್ ಬೆಟ್ಹೋವನ್ - - , - - 루트비히 판 베토벤 - - , - - Бетховен, Людвиг ван - - , - - Ludvigs van Bēthovens - - , - - Лудвиг ван Бетовен - - , - - ബീഥോവൻ - - , - - लुडविग फान बीथोव्हेन - - , - - ဘီသိုဗင် - - , - - लूट्विश् फ़ान् बेट्होफ़न् - - , - - لڈوگ فان بیتھوون - - , - - لیودویګ وان بیتهووین - - , - - Людвіґ ван Бетховен - - , - - லுடுவிக் ஃவான் பேத்தோவன் - - , - - ลูทวิช ฟัน เบทโฮเฟิน - - , - - بىتخوۋىن - - , - - لڈوگ بیتھوون - - , - - לודוויג וואן בעטהאוון - - , - - ਲੁਡਵਿਗ ਵਾਨ ਬੀਥੋਵਨ - - , - - 路德維希·范·貝多芬 - - , - - 路德维希·范·贝多芬 - - , - - Van Beethoveni Ludwig - - , - - Liudvėks van Bethuovens - - , - - Людвіг ван Бэтговэн - - , - - లుడ్విగ్ వాన్ బీథోవెన్ - - , - - लुड्विग भान बिथोभेन - - , - - لودويک بتهوون - - , - - লুডৱিগ ভেন বীথোভেন - - , - - ලූඩ්විග් වැන් බීතෝවන් - - , - - लुडविग वान बीथोवेन - - , - - 路德维希·凡·贝多芬 - - , - - Ludwig Beethoven - - , - - Бетxовен Людвиг ван - - , - - Людвиг ван Бетҳовен - - , - - Լիւտվիկ Վան Պեթհովէն - - , - - بيتهوفن - - , - - لۊدويگ وان بئتهؤون - - , - - Лудвиг ван Бетгьовен - - , - - Lodewèk pan Bèthopen - - - -
-
- ＝ -
- - Aliases: - - Beethoven - - , - - L. van Beethoven - - , - - Ludvig van Beethoven - - , - - L. Bethovenas - - , - - Ludwik van Beethoven - - , - - [ - - 42 more... - - ] - - - - Louis van Beethoven - - , - - L. v. Beethoven - - , - - L. van Betkhoven - - , - - Ludwig von Beethoven - - , - - 贝多芬 - - , - - 路德維希·範·貝多芬 - - , - - 路德維希·凡·貝多芬 - - , - - 贝多芬，L.van - - , - - ルートビッヒ・ベートベン - - , - - ベートーヴェン - - , - - ベートーベン - - , - - 楽聖 - - , - - בטהובן - - , - - בעטהאָוון, לודוויג וואן - - , - - Ludoviko Betoveno - - , - - Людвиг Бетховен - - , - - Бетховен - - , - - Lodewijk van Beethoven - - , - - لودفيغ فان بيتهوفن - - , - - لودويج فان بيتهوفن - - , - - لودويغ فان بيتهوفن - - , - - ludvix.fan.betofn - - , - - Бетовен - - , - - بیتھۆڤن - - , - - بێیتهۆڤن - - , - - Bethoven - - , - - Bethoven Lüdvig van - - , - - Л. ван Бетовен - - , - - பீதோவன் - - , - - பீத்தேவன் - - , - - 베토벤 - - , - - Ljudvig van Betchoven - - , - - 00002507239 IPI - - , - - Ludvig Van Betoven - - , - - बिथोभेन - - , - - Μπετόβεν - - , - - Λ. βαν Μπετόβεν - - , - - Λ. β. Μπετόβεν - - , - - লুটৱিস ফান বিটহ'ফন - - , - - बेट्होफ़न् - - , - - Bèthopen - - , - - เบทโฮเฟิน - - - -
-
- ＝ -
- - Authorities - - - WorldCat - - , - - Wikipedia - - , - - VIAF - - : - - 32182557 - - , - - LCCN - - : - - n79107741 - - , - - ISNI - - : - - 0000000121268987 - - , - - [ - - 17 more... - - ] - - - - GND - - : - - 118508288 - - , - - SELIBR - - : - - 177119 - - , - - SUDOC - - : - - 026714329 - - , - - BNF - - : - - 138912954 - - , - - BIBSYS - - : - - 90051084 - - , - - ULAN - - : - - 500339269 - - , - - MusicBrainz - - : - - 1f9df192-a621-4f54-8850-2c5373b7eac9 - - , - - NLA - - : - - 35016586 - - , - - NDL - - : - - 00432740 - - , - - NCL - - : - - 000441831 - - , - - NKC - - : - - jn19990000607 - - , - - ICCU - - : - - CFIV005324 - - , - - BNE - - : - - XX826607 - - , - - CiNii - - : - - DA00371498 - - , - - KulturNav - - : - - b5001314-26fc-4b3c-bace-4fa56e01e14d - - , - - RKD - - : - - 448562 - - , - - IATH - - : - - w6fn14dt - - - -
-
- ✕ -
- - Show works by type/instrument - -
-
-

- - External links - -

-
- -
-
- -
-
- -
-

- - See also - -

-
- - -
-

- - Miscellaneous information - -

-
-
    -
  • - " - - WoO - - " numbers for works are cited from: Georg Kinsky & Hans Halm, - - Das Werk Beethovens - - (Munich, 1955) -
    • -
  • - " - - H - - " numbers for works are cited from: Willy Hess, - - Verzeichnis der nicht in der Gesamtausgabe veröffentlichen Werke Ludwig van Beethovens - - (Wiesbaden, 1957) -
    • -
-
-
-
-
-
-
-
- -
-
- Table of Contents: - - - Top - - - - - - - A - - - - - B - - - - - C - - - - - D - - - - - E - - - - - F - - - - - G - - - - - H - - - - - I - - - - - J - - - - - K - - - - - L - - - - - M - - - - - N - - - - - O - - - - - P - - - - - Q - - - - - R - - - - - S - - - - - T - - - - - U - - - - - V - - - - - W - - - - - X - - - - - Y - - - - - Z - - - - - - - Chart - - -
-
- - -
-
- -
-

- Compositions (359) -

-
-
-

- Compositions by: Beethoven, Ludwig van -

-

- The following - - 200 - - pages are in this category, out of - - 359 - - total. - - - - - 🔀 - - - 📻 - - -

-
- (no previous) ( - - next 159 - - ) -
-
- - - - - - - -
-

- A -

- -

- B -

- -

- C -

- -

- D -

- -

- E -

- -

- F -

- -

- G -

- -

- H -

- - 		-

- I -

- -

- J -

- -

- K -

- -

- L -

- -

- M -

- -

- N -

- -

- O -

- -

- P -

- -
-
-
- (no previous) ( - - next 159 - - ) -
-
- - -
-
-
-

- Collaborations (1) -

-
-
-

- Collaborations with: Beethoven, Ludwig van -

-

- The following - - 1 - - pages are in this category, out of - - 1 - - total. - - -

-
- - - - - - -
-

- J -

- -
-
-
- - -
-
-
-

- Pasticcios (1) -

-
-
-

- Pasticcios by or with: Beethoven, Ludwig van -

-

- The following - - 1 - - pages are in this category, out of - - 1 - - total. - - -

-
- - - - - - -
-

- M -

- -
-
-
- - -
-
-
-

- Collections (223) -

-
-
-

- Collections by or with: Beethoven, Ludwig van -

-

- The following - - 200 - - pages are in this category, out of - - 223 - - total. - - - - - 🔀 - - - 📻 - - -

-
- (no previous) ( - - next 23 - - ) -
-
- - - - - - - -
-

-

- -

- A -

- -

- B -

- -

- C -

- -

- D -

- -

- E -

- -

- F -

- -

- G -

- -

- H -

- -

- I -

- -

- K -

- -

- L -

- -

- M -

- - 		-

- M cont. -

- -

- N -

- -

- O -

- -

- P -

- -

- Q -

- -

- R -

- -

- S -

- -

- T -

- -
-
-
- (no previous) ( - - next 23 - - ) -
-
- - -
-
-
-

- As Arranger (10) -

-
-
-

- Arrangements by: Beethoven, Ludwig van -

-

- The following - - 10 - - pages are in this category, out of - - 10 - - total. - - - - - 🔀 - - - 📻 - - -

-
- - - - - - - -
-

- C -

- -

- F -

- -

- P -

- - 		-

- R -

- -

- S -

- -

- T -

- -
-
-
- - -
-
-
-

- As Copyist (1) -

-
-
-

- Works copied by: Beethoven, Ludwig van -

-

- The following - - 1 - - pages are in this category, out of - - 1 - - total. - - -

-
- - - - - - -
-

- S -

- -
-
-
- - -
-
-
-

- As Dedicatee (18) -

-
-
-

- Works dedicated to: Beethoven, Ludwig van -

-

- The following - - 18 - - pages are in this category, out of - - 18 - - total. - - - - - 🔀 - - - 📻 - - -

-
- - - - - - - -
-

- C -

- -

- D -

- -

- F -

- -

- G -

- -

- M -

- -

- P -

- - 		-

- P cont. -

- -

- Q -

- -

- S -

- -

- T -

- -

- V -

- -
-
-
- - -
-
-
-

- Books (4) -

-
-
-

- Books by: Beethoven, Ludwig van -

-

- The following - - 4 - - pages are in this category, out of - - 4 - - total. - - - - - 🔀 - - -

-
- - - - - - - -
-

- B -

- -

- L -

- - 		-

- S -

- -

- T -

- -
-
-
- - -
-
-
-
-
- Retrieved from " - - http://imslp.org/index.php?title=Category:Beethoven,_Ludwig_van&oldid=3064478 - - " -
-
-
-
-
-
- - - -
-
- -
-
-
- -
- -
- - - - - - - - - - - - - - - - - - - - - diff --git a/data_ingest/tests/test_imslp_collection_scraper.py b/data_ingest/tests/test_imslp_collection_scraper.py index e3b8f68..d157256 100644 --- a/data_ingest/tests/test_imslp_collection_scraper.py +++ b/data_ingest/tests/test_imslp_collection_scraper.py @@ -1,13 +1,10 @@ import pytest from bs4 import BeautifulSoup -from utils import get_composer_collection_objects +from utils import get_collection_url, get_composer_collection_objects -def test_getcomposer_collection_objects(): - with open("tests/scrape_responses/beethoven_collections_view.html", "r") as file: - html_content = file.read() - soup = BeautifulSoup(html_content, "html.parser") - collection = get_composer_collection_objects(soup) - - assert collection is True +def test_get_composer_collection_objects(): + url = get_collection_url("Beethoven, Ludwig van") + collection_objects = get_composer_collection_objects(url) + assert len(collection_objects) > 200 # pagination diff --git a/data_ingest/tests/test_parse_movements.py b/data_ingest/tests/test_parse_movements.py index b71221f..96dad3f 100644 --- a/data_ingest/tests/test_parse_movements.py +++ b/data_ingest/tests/test_parse_movements.py @@ -20,7 +20,6 @@ def test_parse_piece_sections(): assert result[1]["name"] == "Allegretto" # url - print(result) assert ( result[0]["download_url"] == "https://imslp.org/wiki/Special:ImagefromIndex/309270" @@ -46,6 +45,5 @@ def test_parse_piece_movements(): assert result[1]["name"] == "Scherzo" # url - print(result) assert result[0]["download_url"] is None assert result[1]["download_url"] is None diff --git a/data_ingest/utils/__init__.py b/data_ingest/utils/__init__.py index 25af8bd..644439e 100644 --- a/data_ingest/utils/__init__.py +++ b/data_ingest/utils/__init__.py @@ -1,4 +1,4 @@ -from .imslp_scraping import get_composer_collection_objects +from .imslp_scraping import get_collection_url, get_composer_collection_objects from .movements import parse_movements -__all__ = ["parse_movements", "get_composer_collection_objects"] +__all__ = ["parse_movements", "get_composer_collection_objects", "get_collection_url"] diff --git a/data_ingest/utils/imslp_scraping.py b/data_ingest/utils/imslp_scraping.py index 5bdb513..c6b8e3f 100644 --- a/data_ingest/utils/imslp_scraping.py +++ b/data_ingest/utils/imslp_scraping.py @@ -3,85 +3,115 @@ from urllib.parse import quote, urlparse import requests -from bs4 import BeautifulSoup, Tag +from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.service import Service -from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait from webdriver_manager.chrome import ChromeDriverManager -def get_collection_page(composer_name: str) -> BeautifulSoup: +def get_collection_url(composer_name: str) -> str: base_url = "https://imslp.org/wiki/Category:" # Encode the composer name correctly using quote encoded_name = quote( composer_name, safe="," ) # This will encode the comma and leave it as '%2C' - full_url = f"{base_url}{encoded_name}" - response = requests.get(full_url) - response.raise_for_status() # Ensure successful request - - return BeautifulSoup(response.text, "html.parser") + return f"{base_url}{encoded_name}" def get_composer_collection_objects(base_url: str) -> List[str]: options = Options() - options.add_argument("--headless") # Run in headless mode (no browser UI) + options.add_argument("--headless") driver = webdriver.Chrome( service=Service(ChromeDriverManager().install()), options=options ) - - # Open the URL directly in Selenium - driver.get(base_url) - - collection_objects = [] try: - collection_link = driver.find_element( - By.XPATH, "//a[contains(text(), 'Collections')]" + # Open the URL directly in Selenium + driver.get(base_url) + wait = WebDriverWait( + driver, 10 + ) # Initialize wait object with 10 second timeout + collection_objects = [] + + # Find and click the Collections link + collection_link = wait.until( + EC.element_to_be_clickable( + (By.XPATH, "//a[contains(text(), 'Collections')]") + ) ) - - # Scroll to the element (optional, if the element is out of view) - ActionChains(driver).move_to_element(collection_link).perform() - - print(f"COLLECTION LINK: {collection_link}") - # Click the collection link collection_container_id = urlparse( collection_link.get_attribute("href") ).fragment - - print(collection_container_id) - collection_link.click() - - # Wait for the new content to load after clicking - time.sleep(1) - - page_source = driver.page_source - soup = BeautifulSoup(page_source, "html.parser") - links = soup.find(id=collection_container_id) - if isinstance(links, Tag): - links = links.find_all("a", class_="categorypagelink") - for link in links: - href = link.get("href") - if href: - collection_objects.append(href) - else: - print("not found") - print(collection_objects) - - except Exception as e: - print(f"Error: {e}") - - driver.quit() - - return collection_objects - - -# # Example usage: -# composer_name = "Beethoven, Ludwig van" # Replace with the composer you're looking for -# soup = get_collection_page(composer_name) -url = "https://imslp.org/wiki/Category:Beethoven,_Ludwig_van" -collection_objects = get_composer_collection_objects(url) - -# Print out the collection objects for debugging + driver.execute_script("arguments[0].click();", collection_link) + + # Wait for the initial collection content to load + if collection_container_id and isinstance(collection_container_id, str): + wait.until(EC.presence_of_element_located((By.ID, collection_container_id))) + + while True: + try: + # Wait for the collection links to be present + wait.until( + EC.presence_of_all_elements_located( + ( + By.CSS_SELECTOR, + f"#{collection_container_id} a.categorypagelink", + ) + ) + ) + + # Get the current page's links + links = driver.find_elements( + By.CSS_SELECTOR, f"#{collection_container_id} a.categorypagelink" + ) + + # Extract hrefs + for link in links: + href = link.get_attribute("href") + if href: + collection_objects.append(href) + + # Look for the next page link + try: + next_page = wait.until( + EC.presence_of_element_located( + ( + By.XPATH, + "//a[contains(@class, 'categorypaginglink') and contains(text(), 'next') and not(contains(text(), 'no next'))]", + ) + ) + ) + + # Use JavaScript click instead of regular click + driver.execute_script("arguments[0].click();", next_page) + + # Wait for the page content to update + time.sleep(1) # Short sleep to allow for any animations + + # Wait for the content to change + # We can do this by waiting for the staleness of one of the current links + if links: + try: + wait.until(EC.staleness_of(links[0])) + except Exception : + pass + + except Exception: + break + + except Exception: + break + + finally: + driver.quit() + + return list(set(collection_objects)) + + +# Example usage: +# url = "https://imslp.org/wiki/Category:Beethoven,_Ludwig_van" +# collection_objects = get_composer_collection_objects(url)