facebook scraping: fix post id, stop using story_fbid query param

snarfed · snarfed · commit 44c611d5b613 · 2022-05-24T15:50:02.000-07:00
diff --git a/README.md b/README.md
@@ -297,6 +297,8 @@ Changelog
 * Twitter
   * Trim alt text in line between post preview and creation
   * Correctly trim Twitter alt text
+* Facebook
+  * Scraping: extract post id from `_ft_` query param instead of `story_fbid`, which is now an opaque token that changes regularly. ([facebook-atom#27](https://github.com/snarfed/facebook-atom/issues/27))
 
 ### 4.0 - 2022-03-23
 
diff --git a/granary/facebook.py b/granary/facebook.py
@@ -1853,10 +1853,22 @@ def scraped_to_activities(self, scraped, log_html=False, **kwargs):
         logger.debug('Skipping "Suggested for you"')
         continue
 
-      url = self._sanitize_url(urllib.parse.urljoin(self.BASE_URL, permalink['href']))
+      url = urllib.parse.urljoin(self.BASE_URL, permalink['href'])
       query = urllib.parse.urlparse(url).query
       parsed = urllib.parse.parse_qs(query)
-      post_id = parsed['story_fbid'][0]
+
+      # story_fbid stopped being useful in May 2022, it switched to an opaque
+      # token that changes regularly, even for the same post.
+      # https://github.com/snarfed/facebook-atom/issues/27
+      ft = util.get_first(parsed, '_ft_') or ''
+      for elem in ft.split(':'):
+        if elem.startswith('top_level_post_id.') or elem.startswith('mf_objid.'):
+          post_id = elem.split('.')[1]
+          if post_id:
+            break
+      else:
+        post_id = util.get_first(parsed, 'story_fbid')
+
       owner_id = parsed['id'][0]
 
       author = self._m_html_author(post)
@@ -1923,6 +1935,7 @@ def scraped_to_activities(self, scraped, log_html=False, **kwargs):
           if util.is_int(count_text):
             reactions_count = int(count_text)
 
+      url = self._sanitize_url(url)
       activities.append({
         'objectType': 'activity',
         'verb': 'post',
diff --git a/granary/tests/test_facebook.py b/granary/tests/test_facebook.py
@@ -1202,6 +1202,10 @@ def read_testdata(filename):
   'actor': obj['author'],
   'object': obj,
 } for obj in MBASIC_OBJS]
+MBASIC_FEED_ACTIVITIES = copy.deepcopy(MBASIC_ACTIVITIES)
+MBASIC_FEED_ACTIVITIES[1]['url'] = \
+  MBASIC_FEED_ACTIVITIES[1]['object']['url'] = \
+    'https://www.facebook.com/story.php?story_fbid=456-story&id=212038'
 
 MBASIC_PHOTO_ACTIVITY = {
   'objectType': 'activity',
@@ -1866,7 +1870,7 @@ def test_get_activities_scrape_timeline(self):
     self.mox.ReplayAll()
 
     activities = self.fbscrape.get_activities(user_id='x', group_id=source.SELF)
-    self.assert_equals(MBASIC_ACTIVITIES, activities)
+    self.assert_equals(MBASIC_FEED_ACTIVITIES, activities)
 
   def test_get_activities_scrape_timeline_fetch_replies_likes(self):
     facebook.now_fn().MultipleTimes().AndReturn(datetime(1999, 1, 1))
@@ -3318,7 +3322,7 @@ def test_scraped_to_activities(self):
     self.mox.ReplayAll()
 
     got, _ = self.fb.scraped_to_activities(MBASIC_HTML_TIMELINE)
-    self.assert_equals(MBASIC_ACTIVITIES, got)
+    self.assert_equals(MBASIC_FEED_ACTIVITIES, got)
 
   def test_scraped_to_activities_no_content(self):
     soup = util.parse_html(MBASIC_HTML_TIMELINE)
@@ -3327,7 +3331,7 @@ def test_scraped_to_activities_no_content(self):
     facebook.now_fn().MultipleTimes().AndReturn(datetime(1999, 1, 1))
     self.mox.ReplayAll()
 
-    expected = copy.deepcopy(MBASIC_ACTIVITIES)
+    expected = copy.deepcopy(MBASIC_FEED_ACTIVITIES)
     expected[0]['object']['content'] = '<div class="widePic">\n\n</div>'
     got, _ = self.fb.scraped_to_activities(str(soup))
     self.assert_equals(expected, got)
diff --git a/granary/tests/testdata/facebook.mbasic.feed.html b/granary/tests/testdata/facebook.mbasic.feed.html
@@ -280,7 +280,7 @@ <h3 class="dx dg dy ee">
                     </div>
                     <div class="cm cn">
                       <span class="cl cm" id="like_456">
-                        <a class="dy dz" aria-label="145 reactions, including Like, Love and Care" href="/story.php?story_fbid=456&amp;id=212038&amp;refid=28&amp;_ft_=...">
+                        <a class="dy dz" aria-label="145 reactions, including Like, Love and Care" href="/story.php?story_fbid=456-story&amp;id=212038&amp;refid=28&amp;_ft_=qid.-6283032955...%3Amf_story_key.354173070...%3Atop_level_post_id.456%3Apage_id.2276775...%3Aweight.3133.86...%3Asty.263%3Amf_objid.456">
                           <span class="ea">
                           </span>145
                         </a>
@@ -293,11 +293,11 @@ <h3 class="dx dg dy ee">
                       </span>
                       <span aria-hidden="true"> ·
                       </span>
-                      <a href="/story.php?story_fbid=456&id=212038&refid=8&_ft_=..." class="dz">55 Comments
+                      <a href="/story.php?story_fbid=456-story&id=212038&refid=8&_ft_=qid.-6283032955...%3Amf_story_key.354173070...%3Atop_level_post_id.456%3Apage_id.2276775...%3Aweight.3133.86...%3Asty.263%3Amf_objid.456" class="dz">55 Comments
                       </a>
                       <span aria-hidden="true"> ·
                       </span>
-                      <a href="/story.php?story_fbid=456&amp;id=212038&amp;refid=8&amp;_ft_=...">Full Story
+                      <a href="/story.php?story_fbid=456-story&amp;id=212038&amp;refid=8&amp;_ft_=qid.-6283032955...%3Amf_story_key.354173070...%3Atop_level_post_id.456%3Apage_id.2276775...%3Aweight.3133.86...%3Asty.263%3Amf_objid.456">Full Story
                       </a>
                       <span aria-hidden="true"> ·
                       </span>
