GH-51037: [C++] Replace RapidJSON with simdjson in JSON parser

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -2869,6 +2869,10 @@ function(build_simdjson)
 
   prepare_fetchcontent()
 
+  # Keep simdjson's threading configuration consistent with Arrow's,
+  # which is required for Emscripten where Arrow threading is disabled.
+  set(SIMDJSON_ENABLE_THREADS ${ARROW_ENABLE_THREADING})
+
   # simdjson enables precompiled headers unconditionally.
   # Recompiling simdjson.cpp against it produces differing artifacts
   # Disable precompiled headers to avoid reproducible build failures.

cpp/src/arrow/json/chunker.cc

@@ -32,16 +32,6 @@ namespace arrow {
 namespace json {
 namespace {
 
-// XXX We could try to SIMD-accelerate this routine but it's called only
-// once per chunk and also will presumably examine a minimal amount of bytes.
-int64_t ConsumeWhitespace(std::string_view view) {
-  const auto ws_count = view.find_first_not_of(" \t\r\n");
-  if (ws_count == std::string_view::npos) {
-    return view.size();
-  }
-  return static_cast<int64_t>(ws_count);
-}
-
 // A BoundaryFinder implementation that assumes JSON objects can contain raw newlines,
 // and uses the structural indexes computed by simdjson to delimit them.
 class ParsingBoundaryFinder : public BoundaryFinder {
@@ -156,7 +146,8 @@ class ParsingBoundaryFinder : public BoundaryFinder {
     if (consumed_length > 0) {
       // If we found at least one document, also consume its trailing whitespace
       // to avoid stray bytes at the end of the stream.
-      consumed_length += ConsumeWhitespace(input.substr(consumed_length));
+      consumed_length += internal::ConsumeJsonWhitespace(input.substr(consumed_length),
+                                                         /*trailing=*/false);
     }
     return consumed_length;
   }