title: Beautifulsoup find method returns not subscriptable object
body: <p>I was trying to create a Twitter scraper using beautifulsoup, request, and json. However, when I tried to run the code, it raised the error object is not subscriptable. I checked the lines where the error is located, but I couldn't find what raises the error. Can someone please help? I couldn't fix it.</p>
  <pre><code> File &quot;tweetscraper.py&quot;, line 131, in &lt;module&gt;
      start()
    File &quot;tweetscraper.py&quot;, line 125, in start
      tweets = get_tweets_data(username, soup)
    File &quot;tweetscraper.py&quot;, line 54, in get_tweets_data
      next_pointer = soup.find(&quot;div&quot;, {&quot;class&quot;: &quot;stream-container&quot;})[&quot;data-min-position&quot;]
  TypeError: 'NoneType' object is not subscriptable
  </code></pre>
  <p>Here is my code:</p>
  <pre><code>def get_tweet_text(tweet):
      tweet_text_box = tweet.find(&quot;p&quot;, {&quot;class&quot;: &quot;TweetTextSize TweetTextSize--normal js-tweet-text tweet-text&quot;})
      images_in_tweet_tag = tweet_text_box.find_all(&quot;a&quot;, {&quot;class&quot;: &quot;twitter-timeline-link u-hidden&quot;})
      tweet_text = tweet_text_box.text
      for image_in_tweet_tag in images_in_tweet_tag:
          tweet_text = tweet_text.replace(image_in_tweet_tag.text, '')
  
      return tweet_text
  
  def get_this_page_tweets(soup):
      tweets_list = list()
      tweets = soup.find_all(&quot;li&quot;, {&quot;data-item-type&quot;: &quot;tweet&quot;})
      for tweet in tweets:
          tweet_data = None
          try:
              tweet_data = get_tweet_text(tweet)
          except Exception as e:
              continue
              #ignore if there is any loading or tweet error
  
          if tweet_data:
              tweets_list.append(tweet_data)
              print(&quot;.&quot;, end=&quot;&quot;)
              sys.stdout.flush()
  
      return tweets_list
  
  
  def get_tweets_data(username, soup):
      tweets_list = list()
      tweets_list.extend(get_this_page_tweets(soup))
  
      next_pointer = soup.find(&quot;div&quot;, {&quot;class&quot;: &quot;stream-container&quot;})[&quot;data-min-position&quot;]
  
      while True:
          next_url = &quot;https://twitter.com/i/profiles/show/&quot; + username + \
                     &quot;/timeline/tweets?include_available_features=1&amp;&quot; \
                     &quot;include_entities=1&amp;max_position=&quot; + next_pointer + &quot;&amp;reset_error_state=false&quot;
  
          next_response = None
          try:
              next_response = requests.get(next_url)
          except Exception as e:
              # in case there is some issue with request. None encountered so far.
              print(e)
              return tweets_list
  
          tweets_data = next_response.text
          tweets_obj = json.loads(tweets_data)
          if not tweets_obj[&quot;has_more_items&quot;] and not tweets_obj[&quot;min_position&quot;]:
              # using two checks here bcz in one case has_more_items was false but there were more items
              print(&quot;\nNo more tweets returned&quot;)
              break
          next_pointer = tweets_obj[&quot;min_position&quot;]
          html = tweets_obj[&quot;items_html&quot;]
          soup = BeautifulSoup(html, 'lxml')
          tweets_list.extend(get_this_page_tweets(soup))
  
      return tweets_list
  
  
  # dump final result in a json file
  def dump_data(username, tweets):
      filename = username+&quot;_twitter.json&quot;
      print(&quot;\nDumping data in file &quot; + filename)
      data = dict()
      data[&quot;tweets&quot;] = tweets
      with open(filename, 'w') as fh:
          fh.write(json.dumps(data))
  
      return filename
  
  
  def get_username():
      # if username is not passed
      if len(sys.argv) &lt; 2:
          usage()
      username = sys.argv[1].strip().lower()
      if not username:
          usage()
  
      return username
  
  
  def start(username = None):
      username = get_username()
      url = &quot;http://www.twitter.com/&quot; + username
      print(&quot;\n\nDownloading tweets for &quot; + username)
      response = None
      try:
          response = requests.get(url)
      except Exception as e:
          print(repr(e))
          sys.exit(1)
      
      if response.status_code != 200:
          print(&quot;Non success status code returned &quot;+str(response.status_code))
          sys.exit(1)
  
      soup = BeautifulSoup(response.text, 'lxml')
  
      if soup.find(&quot;div&quot;, {&quot;class&quot;: &quot;errorpage-topbar&quot;}):
          print(&quot;\n\n Error: Invalid username.&quot;)
          sys.exit(1)
  
      tweets = get_tweets_data(username, soup)
      # dump data in a text file
      dump_data(username, tweets)
      print(str(len(tweets))+&quot; tweets dumped.&quot;)
  
  
  start()
  </code></pre>