From fdab03e5c269d02b4d706e07b04287392c86caa2 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 7 Aug 2020 09:24:28 -0700 Subject: [PATCH 01/67] repokitteh: implement retest azp (#12402) Signed-off-by: Lizan Zhou --- ci/repokitteh/modules/azure_pipelines.star | 49 ++++++++++++++++++++++ repokitteh.star | 4 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 ci/repokitteh/modules/azure_pipelines.star diff --git a/ci/repokitteh/modules/azure_pipelines.star b/ci/repokitteh/modules/azure_pipelines.star new file mode 100644 index 000000000000..dc619e06d226 --- /dev/null +++ b/ci/repokitteh/modules/azure_pipelines.star @@ -0,0 +1,49 @@ +load("github.com/repokitteh/modules/lib/utils.star", "react") + +_azp_context_prefix = "ci/azp: " + +def _retry_azp(organization, project, build_id, token): + """Makes an Azure Pipelines Build API request with retry""" + + url = "https://dev.azure.com/{organization}/{project}/_apis/build/builds/{buildId}?retry=true&api-version=5.1".format(organization = organization, project = project, buildId = build_id) + return http(url = url, method = "PATCH", headers = { + "authorization": "Basic " + token, + "content-type": "application/json;odata=verbose", + }) + +def _get_azp_checks(): + github_checks = github.check_list_runs()["check_runs"] + + check_ids = [] + checks = [] + for check in github_checks: + if check["app"]["slug"] == "azure-pipelines" and check["external_id"] not in check_ids: + check_ids.append(check["external_id"]) + checks.append(check) + + return checks + +def _retry(config, comment_id, command): + msgs = "Retrying Azure Pipelines, to retry CircleCI checks, use `/retest-circle`.\n" + checks = _get_azp_checks() + + retried_checks = [] + for check in checks: + name_with_link = "[{}]({})".format(check["name"], check["details_url"]) + if check["status"] != "completed": + msgs += "Cannot retry non-completed check: {}, please wait.\n".format(name_with_link) + elif check["conclusion"] != "failure": + msgs += "Check {} didn't fail.\n".format(name_with_link) + else: + _, build_id, project = check["external_id"].split("|") + _retry_azp("cncf", project, build_id, config["token"]) + retried_checks.append(name_with_link) + + if len(retried_checks) == 0: + react(comment_id, msgs) + else: + react(comment_id, None) + msgs += "Retried failed jobs in: {}".format(", ".join(retried_checks)) + github.issue_create_comment(msgs) + +handlers.command(name = "retry-azp", func = _retry) diff --git a/repokitteh.star b/repokitteh.star index e902d9eae2ea..cf2385c1dfde 100644 --- a/repokitteh.star +++ b/repokitteh.star @@ -4,6 +4,7 @@ use("github.com/repokitteh/modules/assign.star") use("github.com/repokitteh/modules/review.star") use("github.com/repokitteh/modules/wait.star") use("github.com/repokitteh/modules/circleci.star", secret_token=get_secret('circle_token')) +use("github.com/envoyproxy/envoy/ci/repokitteh/modules/azure_pipelines.star", secret_token=get_secret('azp_token')) use( "github.com/envoyproxy/envoy/ci/repokitteh/modules/ownerscheck.star", paths=[ @@ -28,7 +29,8 @@ use( ], ) -alias('retest', 'retry-circle') +alias('retest-circle', 'retry-circle') +alias('retest', 'retry-azp') def _backport(): github.issue_label('backport/review') From 6d9e2ed1f390aec4788068811a1fc30be3944e8c Mon Sep 17 00:00:00 2001 From: danzh Date: Fri, 7 Aug 2020 12:31:42 -0400 Subject: [PATCH 02/67] quiche: update tar (#12525) Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 19 ++ bazel/repository_locations.bzl | 6 +- .../quiche/envoy_quic_dispatcher.cc | 5 +- .../quiche/envoy_quic_dispatcher.h | 1 + .../quiche/platform/flags_list.h | 182 ++++++++++-------- test/extensions/quic_listeners/quiche/BUILD | 1 + .../quiche/active_quic_listener_test.cc | 2 +- .../quiche/envoy_quic_client_session_test.cc | 2 +- .../quiche/envoy_quic_client_stream_test.cc | 2 +- .../quiche/envoy_quic_dispatcher_test.cc | 2 +- .../quiche/envoy_quic_server_session_test.cc | 2 +- .../quiche/envoy_quic_server_stream_test.cc | 3 +- .../integration/quic_http_integration_test.cc | 2 +- 13 files changed, 139 insertions(+), 90 deletions(-) diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 2ec3f85a4e67..50f9f8443c21 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -3731,6 +3731,25 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "quic_test_tools_session_peer_lib", + srcs = [ + "quiche/quic/test_tools/quic_session_peer.cc", + ], + hdrs = [ + "quiche/quic/test_tools/quic_session_peer.h", + ], + copts = quiche_copts, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_packets_lib", + ":quic_core_session_lib", + ":quic_core_utils_lib", + ":quic_platform", + ], +) + envoy_cc_test_library( name = "quic_test_tools_unacked_packet_map_peer_lib", srcs = ["quiche/quic/test_tools/quic_unacked_packet_map_peer.cc"], diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 145da07e7f25..14c8b6d625be 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -410,9 +410,9 @@ DEPENDENCY_REPOSITORIES = dict( cpe = "N/A", ), com_googlesource_quiche = dict( - # Static snapshot of https://quiche.googlesource.com/quiche/+archive/b2b8ff25f5a565324b93411ca29c3403ccbca969.tar.gz - sha256 = "792924bbf27203bb0d1d08c99597a30793ef8f4cfa2df99792aea7200f1b27e3", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/b2b8ff25f5a565324b93411ca29c3403ccbca969.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/quiche/+archive/96bd860bec207d4b722ab7f319fa47be129a85cd.tar.gz + sha256 = "d7129a2f41f2bd00a8a38b33f9b7b955d3e7de3dec20f69b70d7000d3a856360", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/96bd860bec207d4b722ab7f319fa47be129a85cd.tar.gz"], use_category = ["dataplane"], cpe = "N/A", ), diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc index 08564b722580..ba8f7f3a8239 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc @@ -48,8 +48,9 @@ void EnvoyQuicDispatcher::OnConnectionClosed(quic::QuicConnectionId connection_i } std::unique_ptr EnvoyQuicDispatcher::CreateQuicSession( - quic::QuicConnectionId server_connection_id, const quic::QuicSocketAddress& peer_address, - quiche::QuicheStringPiece /*alpn*/, const quic::ParsedQuicVersion& version) { + quic::QuicConnectionId server_connection_id, const quic::QuicSocketAddress& /*self_address*/, + const quic::QuicSocketAddress& peer_address, quiche::QuicheStringPiece /*alpn*/, + const quic::ParsedQuicVersion& version) { auto quic_connection = std::make_unique( server_connection_id, peer_address, *helper(), *alarm_factory(), writer(), /*owns_writer=*/false, quic::ParsedQuicVersionVector{version}, listen_socket_); diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h index ede0c5b42625..5921342b84bf 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h @@ -59,6 +59,7 @@ class EnvoyQuicDispatcher : public quic::QuicDispatcher { protected: std::unique_ptr CreateQuicSession(quic::QuicConnectionId server_connection_id, + const quic::QuicSocketAddress& self_address, const quic::QuicSocketAddress& peer_address, quiche::QuicheStringPiece alpn, const quic::ParsedQuicVersion& version) override; diff --git a/source/extensions/quic_listeners/quiche/platform/flags_list.h b/source/extensions/quic_listeners/quiche/platform/flags_list.h index 776521f42d0d..587e80054c0a 100644 --- a/source/extensions/quic_listeners/quiche/platform/flags_list.h +++ b/source/extensions/quic_listeners/quiche/platform/flags_list.h @@ -17,6 +17,10 @@ QUICHE_FLAG( bool, http2_reloadable_flag_http2_backend_alpn_failure_error_code, false, "If true, the GFE will return a new ResponseCodeDetails error when ALPN to the backend fails.") +QUICHE_FLAG(bool, http2_reloadable_flag_http2_ip_based_cwnd_exp, false, + "If true, enable IP address based CWND bootstrapping experiment with different " + "bandwidth models and priorities in HTTP2.") + QUICHE_FLAG(bool, http2_reloadable_flag_http2_security_requirement_for_client3, false, "If true, check whether client meets security requirements during SSL handshake. If " "flag is true and client does not meet security requirements, do not negotiate HTTP/2 " @@ -31,10 +35,18 @@ QUICHE_FLAG(bool, quic_reloadable_flag_advertise_quic_for_https_for_debugips, fa QUICHE_FLAG(bool, quic_reloadable_flag_advertise_quic_for_https_for_external_users, false, "") +QUICHE_FLAG(bool, quic_reloadable_flag_gclb_quic_allow_alia, true, + "If gfe2_reloadable_flag_gclb_use_alia is also true, use Alia for GCLB QUIC " + "handshakes. To be used as a big red button if there's a problem with Alia/QUIC.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_ack_delay_alarm_granularity, false, "When true, ensure the ACK delay is never less than the alarm granularity when ACK " "decimation is enabled.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_add_silent_idle_timeout, false, + "If true, when server is silently closing connections due to idle timeout, serialize " + "the connection close packets which will be added to time wait list.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_allow_backend_set_stream_ttl, false, "If true, check backend response header for X-Response-Ttl. If it is provided, the " "stream TTL is set. A QUIC stream will be immediately canceled when tries to write " @@ -46,17 +58,6 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_allow_client_enabled_bbr_v2, true, QUICHE_FLAG(bool, quic_reloadable_flag_quic_alpn_dispatch, false, "Support different QUIC sessions, as indicated by ALPN. Used for QBONE.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_always_send_earliest_ack, false, - "If true, SendAllPendingAcks always send the earliest ACK.") - -QUICHE_FLAG( - bool, quic_reloadable_flag_quic_avoid_leak_writer_buffer, true, - "If true, QUIC will free writer-allocated packet buffer if writer->WritePacket is not called.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr2_add_ack_height_to_queueing_threshold, true, - "If true, QUIC BBRv2 to take ack height into account when calculating " - "queuing_threshold in PROBE_UP.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr2_avoid_too_low_probe_bw_cwnd, false, "If true, QUIC BBRv2's PROBE_BW mode will not reduce cwnd below BDP+ack_height.") @@ -67,31 +68,36 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr2_fewer_startup_round_trips, fals QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr2_ignore_inflight_lo, false, "When true, QUIC's BBRv2 ignores inflight_lo in PROBE_BW.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr2_improve_adjust_network_parameters, false, + "If true, improve Bbr2Sender::AdjustNetworkParameters by 1) do not inject a bandwidth " + "sample to the bandwidth filter, and 2) re-calculate pacing rate after cwnd updated..") + QUICHE_FLAG( bool, quic_reloadable_flag_quic_bbr2_limit_inflight_hi, false, "When true, the B2HI connection option limits reduction of inflight_hi to (1-Beta)*CWND.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_donot_inject_bandwidth, true, - "If true, do not inject bandwidth in BbrSender::AdjustNetworkParameters.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_fix_pacing_rate, true, - "If true, re-calculate pacing rate when cwnd gets bootstrapped.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_flexible_app_limited, false, "When true and the BBR9 connection option is present, BBR only considers bandwidth " "samples app-limited if they're not filling the pipe.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_mitigate_overly_large_bandwidth_sample, true, - "If true, when cwnd gets bootstrapped and causing badly overshoot, reset cwnd and " - "pacing rate based on measured bw.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_no_bytes_acked_in_startup_recovery, false, "When in STARTUP and recovery, do not add bytes_acked to QUIC BBR's CWND in " "CalculateCongestionWindow()") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_bootstrap_cwnd_by_gfe_bandwidth, false, + "If true, bootstrap initial QUIC cwnd by GFE measured bandwidth models.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_bootstrap_cwnd_by_spdy_priority, true, "If true, bootstrap initial QUIC cwnd by SPDY priorities.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_check_encryption_level_in_fast_path, false, + "If true, when data is sending in fast path mode in the creator, making sure stream " + "data is sent in the right encryption level.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_coalesced_packet_of_higher_space2, false, + "If true, try to coalesce packet of higher space with retransmissions to mitigate RTT " + "inflations.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_bursts, false, "If true, set burst token to 2 in cwnd bootstrapping experiment.") @@ -114,24 +120,24 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_default_to_bbr_v2, false, QUICHE_FLAG(bool, quic_reloadable_flag_quic_determine_serialized_packet_fate_early, false, "If true, determine a serialized packet's fate before the packet gets serialized.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_draft_25, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_server_blackhole_detection, false, + "If true, disable blackhole detection on server side.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_draft_25, true, "If true, disable QUIC version h3-25.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_draft_27, false, "If true, disable QUIC version h3-27.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_draft_29, false, + "If true, disable QUIC version h3-29.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_q043, false, "If true, disable QUIC version Q043.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_q046, false, "If true, disable QUIC version Q046.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_q048, true, - "If true, disable QUIC version Q048.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_q049, true, - "If true, disable QUIC version Q049.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_q050, false, "If true, disable QUIC version Q050.") @@ -150,66 +156,66 @@ QUICHE_FLAG( bool, quic_reloadable_flag_quic_do_not_close_stream_again_on_connection_close, false, "If true, do not try to close stream again if stream fails to be closed upon connection close.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_do_not_use_stream_map, false, + "If true, QUIC subclasses will no longer directly access stream_map for its content.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_donot_reset_ideal_next_packet_send_time, false, "If true, stop resetting ideal_next_packet_send_time_ in pacing sender.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_dont_pad_chlo, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_dont_pad_chlo, true, "When true, do not pad the QUIC_CRYPTO CHLO message itself. Note that the packet " "containing the CHLO will still be padded.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_dont_send_max_ack_delay_if_default, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_dont_send_max_ack_delay_if_default, true, "When true, QUIC_CRYPTO versions of QUIC will not send the max ACK delay unless it is " "configured to a non-default value.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_ack_decimation, true, - "Default enables QUIC ack decimation and adds a connection option to disable it.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_loss_detection_experiment_at_gfe, false, "If ture, enable GFE-picked loss detection experiment.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_loss_detection_tuner, false, "If true, allow QUIC loss detection tuning to be enabled by connection option ELDT.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_tls_resumption_v2, false, - "If true, enables support for TLS resumption in QUIC.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_draft_29, false, - "If true, enable QUIC version h3-29.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_overshooting_detection, false, + "If true, enable overshooting detection when the DTOS connection option is supplied.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_zero_rtt_for_tls, false, - "If true, support for IETF QUIC 0-rtt is enabled.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_t051, false, + "If true, enable QUIC version h3-T051.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_enabled, false, "") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_bbr_cwnd_in_bandwidth_resumption, true, - "If true, adjust congestion window when doing bandwidth resumption in BBR.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_extra_padding_bytes, false, "If true, consider frame expansion when calculating extra padding bytes to meet " "minimum plaintext packet size required for header protection.") -QUICHE_FLAG( - bool, quic_reloadable_flag_quic_fix_gquic_stream_type, false, - "If true, do not use QuicUtil::IsBidirectionalStreamId() to determine gQUIC stream type.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_min_crypto_frame_size, true, - "If true, include MinPlaintextPacketSize when deterine whether removing soft limit for " - "crypto frames.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_neuter_handshake_data, false, + "If true, fix a case where data is marked lost in HANDSHAKE level but HANDSHAKE key " + "gets decrypted later.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_packet_number_length, false, "If true, take the largest acked packet into account when computing the sent packet " "number length.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_pto_timeout, true, - "If true, use 0 as ack_delay when calculate PTO timeout for INITIAL and HANDSHAKE " - "packet number space.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_print_draft_version, false, + "When true, ParsedQuicVersionToString will print IETF drafts with format draft29 " + "instead of ff00001d.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_server_pto_timeout, true, - "If true, do not arm PTO on half RTT packets if they are the only ones in flight.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_get_stream_information_from_stream_map, false, + "If true, gQUIC will only consult stream_map in QuicSession::GetNumActiveStreams().") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_undecryptable_packets, false, - "If true, remove the head of line blocking caused by an unprocessable packet in the " - "undecryptable packets list.") +QUICHE_FLAG( + bool, quic_reloadable_flag_quic_http3_goaway_new_behavior, false, + "If true, server accepts GOAWAY (draft-28 behavior), client receiving GOAWAY with stream ID " + "that is not client-initiated bidirectional stream ID closes connection with H3_ID_ERROR " + "(draft-28 behavior). Also, receiving a GOAWAY with ID larger than previously received closes " + "connection with H3_ID_ERROR. If false, server receiving GOAWAY closes connection with " + "H3_FRAME_UNEXPECTED (draft-27 behavior), client receiving GOAWAY with stream ID that is not " + "client-initiated bidirectional stream ID closes connection with PROTOCOL_VIOLATION (draft-04 " + "behavior), larger ID than previously received does not trigger connection close.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_ip_based_cwnd_exp, false, + "If true, enable IP address based CWND bootstrapping experiment with different " + "bandwidth models and priorities. ") QUICHE_FLAG(bool, quic_reloadable_flag_quic_listener_never_fake_epollout, false, "If true, QuicListener::OnSocketIsWritable will always return false, which means there " @@ -230,15 +236,43 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_record_frontend_service_vip_mapping, "If true, for L1 GFE, as requests come in, record frontend service to VIP mapping " "which is used to announce VIP in SHLO for proxied sessions. ") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_record_received_min_ack_delay, false, + "If true, record the received min_ack_delay in transport parameters to QUIC config.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_reject_all_traffic, false, "") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_remove_streams_waiting_for_acks, false, + "If true, QuicSession will no longer need streams_waiting_for_acks_.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_remove_unused_ack_options, false, + "Remove ACK_DECIMATION_WITH_REORDERING mode and fast_ack_after_quiescence option in " + "QUIC received packet manager.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_remove_zombie_streams, false, + "If true, QuicSession doesn't keep a separate zombie_streams. Instead, all streams are " + "stored in stream_map_.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_require_handshake_confirmation, false, "If true, require handshake confirmation for QUIC connections, functionally disabling " "0-rtt handshakes.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_save_user_agent_in_quic_session, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_retransmit_handshake_data_early, false, + "If true, retransmit unacked handshake data before PTO expiry.") + +QUICHE_FLAG( + bool, quic_reloadable_flag_quic_revert_mtu_after_two_ptos, false, + "If true, QUIC connection will revert to a previously validated MTU(if exists) after two PTOs.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_save_user_agent_in_quic_session, true, "If true, save user agent into in QuicSession.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_early_data_header_to_backend, false, + "If true, for 0RTT IETF QUIC requests, GFE will append a Early-Data header and send it " + "to backend.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_path_response, false, + "If true, send PATH_RESPONSE upon receiving PATH_CHALLENGE regardless of perspective.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_timestamps, false, "When the STMP connection option is sent by the client, timestamps in the QUIC ACK " "frame are sent and processed.") @@ -246,10 +280,10 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_timestamps, false, QUICHE_FLAG(bool, quic_reloadable_flag_quic_server_push, true, "If true, enable server push feature on QUIC.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_stop_sending_duplicate_max_streams, false, - "If true, session does not send duplicate MAX_STREAMS.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_simplify_received_packet_manager_ack, false, + "Simplify the ACK code in quic_received_packet_manager.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_support_handshake_done_in_t050, false, +QUICHE_FLAG(bool, quic_reloadable_flag_quic_support_handshake_done_in_t050, true, "If true, support HANDSHAKE_DONE frame in T050.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_testonly_default_false, false, @@ -265,9 +299,8 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_unified_iw_options, false, QUICHE_FLAG(bool, quic_reloadable_flag_quic_update_packet_size, false, "If true, update packet size when the first frame gets queued.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_dispatcher_clock_for_read_timestamp, true, - "If true, in QuicListener, use QuicDispatcher's clock as the source for packet read " - "timestamps.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_half_rtt_as_first_pto, false, + "If true, when TLPR copt is used, enable half RTT as first PTO timeout.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_header_stage_idle_list2, false, "If true, use header stage idle list for QUIC connections in GFE.") @@ -294,19 +327,16 @@ QUICHE_FLAG( bool, quic_restart_flag_quic_allow_loas_multipacket_chlo, false, "If true, inspects QUIC CHLOs for kLOAS and early creates sessions to allow multi-packet CHLOs") -QUICHE_FLAG(bool, quic_restart_flag_quic_dispatcher_track_top_1k_client_ip, true, - "If true, GfeQuicDispatcher will track the top 1000 client IPs.") +QUICHE_FLAG(bool, quic_restart_flag_quic_enable_tls_resumption_v4, false, + "If true, enables support for TLS resumption in QUIC.") + +QUICHE_FLAG(bool, quic_restart_flag_quic_enable_zero_rtt_for_tls_v2, false, + "If true, support for IETF QUIC 0-rtt is enabled.") -QUICHE_FLAG(bool, quic_restart_flag_quic_google_transport_param_omit_old, false, +QUICHE_FLAG(bool, quic_restart_flag_quic_google_transport_param_omit_old, true, "When true, QUIC+TLS will not send nor parse the old-format Google-specific transport " "parameters.") -QUICHE_FLAG(bool, quic_restart_flag_quic_ignore_cid_first_byte_in_rx_ring_bpf, true, - "If true, ignore CID first byte in BPF for RX_RING.") - -QUICHE_FLAG(bool, quic_restart_flag_quic_memslice_ensure_ownership, true, - "Call gfe2::MemSlice::EnsureReferenceCounted in the constructor of QuicMemSlice.") - QUICHE_FLAG(bool, quic_restart_flag_quic_offload_pacing_to_usps2, false, "If true, QUIC offload pacing when using USPS as egress method.") @@ -334,10 +364,6 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_use_pigeon_socket_to_backend, false, "If true, create a shared pigeon socket for all quic to backend connections and switch " "to use it after successful handshake.") -QUICHE_FLAG(bool, spdy_reloadable_flag_fix_spdy_header_coalescing, true, - "If true, when coalescing multivalued spdy headers, only headers that exist in spdy " - "headers block are updated.") - QUICHE_FLAG(bool, spdy_reloadable_flag_quic_bootstrap_cwnd_by_spdy_priority, true, "If true, bootstrap initial QUIC cwnd by SPDY priorities.") diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index e14bc1f36fef..b3c4eb70698d 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -77,6 +77,7 @@ envoy_cc_test( "//test/mocks/network:network_mocks", "//test/test_common:utility_lib", "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", + "@com_googlesource_quiche//:quic_test_tools_session_peer_lib", ], ) diff --git a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc index 8c1e7e222790..747452ccdf78 100644 --- a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc +++ b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc @@ -85,7 +85,7 @@ class ActiveQuicListenerTest : public QuicMultiVersionTest { return quic::CurrentSupportedVersionsWithQuicCrypto(); } bool use_http3 = GetParam().second == QuicVersionType::Iquic; - SetQuicReloadableFlag(quic_enable_version_draft_29, use_http3); + SetQuicReloadableFlag(quic_disable_version_draft_29, !use_http3); SetQuicReloadableFlag(quic_disable_version_draft_27, !use_http3); SetQuicReloadableFlag(quic_disable_version_draft_25, !use_http3); return quic::CurrentSupportedVersions(); diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_client_session_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_client_session_test.cc index 5db43230cd7c..488fe023354e 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_client_session_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_client_session_test.cc @@ -95,7 +95,7 @@ class EnvoyQuicClientSessionTest : public testing::TestWithParam { : api_(Api::createApiForTest(time_system_)), dispatcher_(api_->allocateDispatcher("test_thread")), connection_helper_(*dispatcher_), alarm_factory_(*dispatcher_, *connection_helper_.GetClock()), quic_version_([]() { - SetQuicReloadableFlag(quic_enable_version_draft_29, GetParam()); + SetQuicReloadableFlag(quic_disable_version_draft_29, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_27, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_25, !GetParam()); return quic::ParsedVersionOfIndex(quic::CurrentSupportedVersions(), 0); diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_client_stream_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_client_stream_test.cc index 2a32df6319ed..9784c7231ff2 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_client_stream_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_client_stream_test.cc @@ -25,7 +25,7 @@ class EnvoyQuicClientStreamTest : public testing::TestWithParam { : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")), connection_helper_(*dispatcher_), alarm_factory_(*dispatcher_, *connection_helper_.GetClock()), quic_version_([]() { - SetQuicReloadableFlag(quic_enable_version_draft_29, GetParam()); + SetQuicReloadableFlag(quic_disable_version_draft_29, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_27, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_25, !GetParam()); return quic::CurrentSupportedVersions()[0]; diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc index c3ab38f57ff5..fb15815fa1db 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc @@ -65,7 +65,7 @@ class EnvoyQuicDispatcherTest : public QuicMultiVersionTest, return quic::CurrentSupportedVersionsWithQuicCrypto(); } bool use_http3 = GetParam().second == QuicVersionType::Iquic; - SetQuicReloadableFlag(quic_enable_version_draft_29, use_http3); + SetQuicReloadableFlag(quic_disable_version_draft_29, !use_http3); SetQuicReloadableFlag(quic_disable_version_draft_27, !use_http3); SetQuicReloadableFlag(quic_disable_version_draft_25, !use_http3); return quic::CurrentSupportedVersions(); diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc index 6ddae3c80624..f2ef9fae069e 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc @@ -145,7 +145,7 @@ class EnvoyQuicServerSessionTest : public testing::TestWithParam { : api_(Api::createApiForTest(time_system_)), dispatcher_(api_->allocateDispatcher("test_thread")), connection_helper_(*dispatcher_), alarm_factory_(*dispatcher_, *connection_helper_.GetClock()), quic_version_([]() { - SetQuicReloadableFlag(quic_enable_version_draft_29, GetParam()); + SetQuicReloadableFlag(quic_disable_version_draft_29, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_27, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_25, !GetParam()); return quic::ParsedVersionOfIndex(quic::CurrentSupportedVersions(), 0); diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc index 6468f95fe9fa..4a4236737bd0 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc @@ -7,6 +7,7 @@ #pragma GCC diagnostic ignored "-Winvalid-offsetof" #include "quiche/quic/test_tools/quic_connection_peer.h" +#include "quiche/quic/test_tools/quic_session_peer.h" #pragma GCC diagnostic pop #include "common/event/libevent_scheduler.h" @@ -39,7 +40,7 @@ class EnvoyQuicServerStreamTest : public testing::TestWithParam { : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")), connection_helper_(*dispatcher_), alarm_factory_(*dispatcher_, *connection_helper_.GetClock()), quic_version_([]() { - SetQuicReloadableFlag(quic_enable_version_draft_29, GetParam()); + SetQuicReloadableFlag(quic_disable_version_draft_29, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_27, !GetParam()); SetQuicReloadableFlag(quic_disable_version_draft_25, !GetParam()); return quic::CurrentSupportedVersions()[0]; diff --git a/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc b/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc index bbe34b658e7b..05fb1e61a7aa 100644 --- a/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc +++ b/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc @@ -54,7 +54,7 @@ class QuicHttpIntegrationTest : public HttpIntegrationTest, public QuicMultiVers return quic::CurrentSupportedVersionsWithQuicCrypto(); } bool use_http3 = GetParam().second == QuicVersionType::Iquic; - SetQuicReloadableFlag(quic_enable_version_draft_29, use_http3); + SetQuicReloadableFlag(quic_disable_version_draft_29, !use_http3); SetQuicReloadableFlag(quic_disable_version_draft_27, !use_http3); SetQuicReloadableFlag(quic_disable_version_draft_25, !use_http3); return quic::CurrentSupportedVersions(); From e0bd39855d03534afa28c90d2a1a00c08c837c3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A5=81=E6=97=A0=E5=BF=A7?= Date: Sat, 8 Aug 2020 00:32:39 +0800 Subject: [PATCH 03/67] docs: Update x-envoy-upstream-service-time description to be more accurate (#12517) Signed-off-by: wbpcode --- docs/root/configuration/http/http_filters/router_filter.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/root/configuration/http/http_filters/router_filter.rst b/docs/root/configuration/http/http_filters/router_filter.rst index 0aaa931891e9..4ca285e9eda7 100644 --- a/docs/root/configuration/http/http_filters/router_filter.rst +++ b/docs/root/configuration/http/http_filters/router_filter.rst @@ -360,9 +360,9 @@ HTTP response headers set on downstream responses x-envoy-upstream-service-time ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Contains the time in milliseconds spent by the upstream host processing the request. This is useful -if the client wants to determine service time compared to network latency. This header is set on -responses. +Contains the time in milliseconds spent by the upstream host processing the request and the network +latency between Envoy and upstream host. This is useful if the client wants to determine service time +compared to network latency between client and Envoy. This header is set on responses. .. _config_http_filters_router_x-envoy-overloaded_set: From 43b110ab6ec17c80463a50bf6d3ae6077f9fb226 Mon Sep 17 00:00:00 2001 From: danzh Date: Fri, 7 Aug 2020 13:16:29 -0400 Subject: [PATCH 04/67] quiche: implement certificate verification (#12063) Implement quic::ProofVerifier which consists of cert verification and signature verification. Cert verification: Share cert verification code with Extensions::TransportSockets::Tls::ClientContextImpl. And initialize ProofVerifier using Envoy::Ssl::ClientContextConfig protobuf. Signature verification: Use quic::CertificateViewer to verify signature. Part of #9434 #2557 Signed-off-by: Dan Zhang --- source/extensions/quic_listeners/quiche/BUILD | 36 ++- .../quiche/envoy_quic_fake_proof_verifier.h | 61 ----- .../quiche/envoy_quic_proof_source.cc | 15 +- .../quiche/envoy_quic_proof_source.h | 19 +- .../quiche/envoy_quic_proof_source_base.cc | 81 ++++++ ...ource.h => envoy_quic_proof_source_base.h} | 44 +-- .../quiche/envoy_quic_proof_verifier.cc | 48 ++++ .../quiche/envoy_quic_proof_verifier.h | 30 +++ .../quiche/envoy_quic_proof_verifier_base.cc | 70 +++++ .../quiche/envoy_quic_proof_verifier_base.h | 47 ++++ .../quic_listeners/quiche/envoy_quic_utils.cc | 61 +++++ .../quic_listeners/quiche/envoy_quic_utils.h | 11 + .../tls/context_config_impl.h | 5 +- .../transport_sockets/tls/context_impl.cc | 79 ++++-- .../transport_sockets/tls/context_impl.h | 9 +- test/extensions/quic_listeners/quiche/BUILD | 27 +- .../quiche/crypto_test_utils_for_envoy.cc | 4 +- .../quiche/envoy_quic_proof_source_test.cc | 221 ++++++++++++--- .../quiche/envoy_quic_proof_verifier_test.cc | 252 ++++++++++++++++++ .../integration/quic_http_integration_test.cc | 75 +++++- .../quic_listeners/quiche/test_proof_source.h | 20 +- .../quiche/test_proof_verifier.h | 30 +++ test/mocks/ssl/mocks.h | 17 ++ tools/spelling/spelling_dictionary.txt | 1 + 24 files changed, 1072 insertions(+), 191 deletions(-) delete mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_proof_source_base.cc rename source/extensions/quic_listeners/quiche/{envoy_quic_fake_proof_source.h => envoy_quic_proof_source_base.h} (68%) create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.h create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_test.cc create mode 100644 test/extensions/quic_listeners/quiche/test_proof_verifier.h diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 1099eb26deb8..fd2cce9b9b5f 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -62,12 +62,17 @@ envoy_cc_library( ) envoy_cc_library( - name = "envoy_quic_fake_proof_source_lib", - hdrs = ["envoy_quic_fake_proof_source.h"], + name = "envoy_quic_proof_source_base_lib", + srcs = ["envoy_quic_proof_source_base.cc"], + hdrs = ["envoy_quic_proof_source_base.h"], external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ + ":envoy_quic_utils_lib", + "@com_googlesource_quiche//:quic_core_crypto_certificate_view_lib", + "@com_googlesource_quiche//:quic_core_crypto_crypto_handshake_lib", "@com_googlesource_quiche//:quic_core_crypto_proof_source_interface_lib", + "@com_googlesource_quiche//:quic_core_data_lib", "@com_googlesource_quiche//:quic_core_versions_lib", ], ) @@ -79,7 +84,7 @@ envoy_cc_library( external_deps = ["ssl"], tags = ["nofips"], deps = [ - ":envoy_quic_fake_proof_source_lib", + ":envoy_quic_proof_source_base_lib", ":envoy_quic_utils_lib", ":quic_io_handle_wrapper_lib", ":quic_transport_socket_factory_lib", @@ -91,16 +96,32 @@ envoy_cc_library( ) envoy_cc_library( - name = "envoy_quic_proof_verifier_lib", - hdrs = ["envoy_quic_fake_proof_verifier.h"], + name = "envoy_quic_proof_verifier_base_lib", + srcs = ["envoy_quic_proof_verifier_base.cc"], + hdrs = ["envoy_quic_proof_verifier_base.h"], external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ + ":envoy_quic_utils_lib", + "@com_googlesource_quiche//:quic_core_crypto_certificate_view_lib", "@com_googlesource_quiche//:quic_core_crypto_crypto_handshake_lib", "@com_googlesource_quiche//:quic_core_versions_lib", ], ) +envoy_cc_library( + name = "envoy_quic_proof_verifier_lib", + srcs = ["envoy_quic_proof_verifier.cc"], + hdrs = ["envoy_quic_proof_verifier.h"], + external_deps = ["quiche_quic_platform"], + tags = ["nofips"], + deps = [ + ":envoy_quic_proof_verifier_base_lib", + ":envoy_quic_utils_lib", + "//source/extensions/transport_sockets/tls:context_lib", + ], +) + envoy_cc_library( name = "spdy_server_push_utils_for_envoy_lib", srcs = ["spdy_server_push_utils_for_envoy.cc"], @@ -323,7 +344,10 @@ envoy_cc_library( name = "envoy_quic_utils_lib", srcs = ["envoy_quic_utils.cc"], hdrs = ["envoy_quic_utils.h"], - external_deps = ["quiche_quic_platform"], + external_deps = [ + "quiche_quic_platform", + "ssl", + ], tags = ["nofips"], deps = [ "//include/envoy/http:codec_interface", diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h deleted file mode 100644 index af107983317b..000000000000 --- a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h +++ /dev/null @@ -1,61 +0,0 @@ -#pragma once - -#include "absl/strings/str_cat.h" - -#pragma GCC diagnostic push - -// QUICHE allows unused parameters. -#pragma GCC diagnostic ignored "-Wunused-parameter" - -#include "quiche/quic/core/crypto/proof_verifier.h" -#include "quiche/quic/core/quic_versions.h" - -#pragma GCC diagnostic pop - -namespace Envoy { -namespace Quic { - -// A fake implementation of quic::ProofVerifier which approves the certs and -// signature produced by EnvoyQuicFakeProofSource. -class EnvoyQuicFakeProofVerifier : public quic::ProofVerifier { -public: - ~EnvoyQuicFakeProofVerifier() override = default; - - // quic::ProofVerifier - // Return success if the certs chain is valid and signature is "Fake signature for { - // [server_config] }". Otherwise failure. - quic::QuicAsyncStatus - VerifyProof(const std::string& hostname, const uint16_t port, - const std::string& /*server_config*/, quic::QuicTransportVersion /*quic_version*/, - absl::string_view /*chlo_hash*/, const std::vector& certs, - const std::string& cert_sct, const std::string& /*signature*/, - const quic::ProofVerifyContext* context, std::string* error_details, - std::unique_ptr* details, - std::unique_ptr callback) override { - if (VerifyCertChain(hostname, port, certs, "", cert_sct, context, error_details, details, - std::move(callback)) == quic::QUIC_SUCCESS) { - return quic::QUIC_SUCCESS; - } - return quic::QUIC_FAILURE; - } - - // Return success upon one arbitrary cert content. Otherwise failure. - quic::QuicAsyncStatus - VerifyCertChain(const std::string& /*hostname*/, const uint16_t /*port*/, - const std::vector& certs, const std::string& /*ocsp_response*/, - const std::string& cert_sct, const quic::ProofVerifyContext* /*context*/, - std::string* /*error_details*/, - std::unique_ptr* /*details*/, - std::unique_ptr /*callback*/) override { - // Cert SCT support is not enabled for fake ProofSource. - if (cert_sct.empty() && certs.size() == 1) { - return quic::QUIC_SUCCESS; - } - return quic::QUIC_FAILURE; - } - - std::unique_ptr CreateDefaultContext() override { return nullptr; } -}; - -} // namespace Quic -} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc index 66fe7017436d..96fe056e818e 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc @@ -28,19 +28,13 @@ EnvoyQuicProofSource::GetCertChain(const quic::QuicSocketAddress& server_address } auto& cert_config = cert_config_ref.value().get(); const std::string& chain_str = cert_config.certificateChain(); - std::string pem_str = std::string(const_cast(chain_str.data()), chain_str.size()); std::stringstream pem_stream(chain_str); std::vector chain = quic::CertificateView::LoadPemFromStream(&pem_stream); - if (chain.empty()) { - ENVOY_LOG(warn, "Failed to load certificate chain from %s", cert_config.certificateChainPath()); - return quic::QuicReferenceCountedPointer( - new quic::ProofSource::Chain({})); - } return quic::QuicReferenceCountedPointer( new quic::ProofSource::Chain(chain)); } -void EnvoyQuicProofSource::ComputeTlsSignature( +void EnvoyQuicProofSource::signPayload( const quic::QuicSocketAddress& server_address, const quic::QuicSocketAddress& client_address, const std::string& hostname, uint16_t signature_algorithm, quiche::QuicheStringPiece in, std::unique_ptr callback) { @@ -59,7 +53,11 @@ void EnvoyQuicProofSource::ComputeTlsSignature( std::stringstream pem_str(pkey); std::unique_ptr pem_key = quic::CertificatePrivateKey::LoadPemFromStream(&pem_str); - + if (pem_key == nullptr) { + ENVOY_LOG(warn, "Failed to load private key."); + callback->Run(false, "", nullptr); + return; + } // Sign. std::string sig = pem_key->Sign(in, signature_algorithm); @@ -85,7 +83,6 @@ EnvoyQuicProofSource::getTlsCertConfigAndFilterChain(const quic::QuicSocketAddre const Network::FilterChain* filter_chain = filter_chain_manager_.findFilterChain(connection_socket); if (filter_chain == nullptr) { - ENVOY_LOG(warn, "No matching filter chain found for handshake."); listener_stats_.no_filter_chain_match_.inc(); return {absl::nullopt, absl::nullopt}; } diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.h b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.h index 4dab673687d8..6e1c74c9234c 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.h @@ -2,14 +2,14 @@ #include "server/connection_handler_impl.h" -#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h" +#include "extensions/quic_listeners/quiche/envoy_quic_proof_source_base.h" #include "extensions/quic_listeners/quiche/quic_transport_socket_factory.h" namespace Envoy { namespace Quic { -class EnvoyQuicProofSource : public EnvoyQuicFakeProofSource, - protected Logger::Loggable { +// A ProofSource implementation which supplies a proof instance with certs from filter chain. +class EnvoyQuicProofSource : public EnvoyQuicProofSourceBase { public: EnvoyQuicProofSource(Network::Socket& listen_socket, Network::FilterChainManager& filter_chain_manager, @@ -19,14 +19,17 @@ class EnvoyQuicProofSource : public EnvoyQuicFakeProofSource, ~EnvoyQuicProofSource() override = default; + // quic::ProofSource quic::QuicReferenceCountedPointer GetCertChain(const quic::QuicSocketAddress& server_address, const quic::QuicSocketAddress& client_address, const std::string& hostname) override; - void ComputeTlsSignature(const quic::QuicSocketAddress& server_address, - const quic::QuicSocketAddress& client_address, - const std::string& hostname, uint16_t signature_algorithm, - quiche::QuicheStringPiece in, - std::unique_ptr callback) override; + +protected: + // quic::ProofSource + void signPayload(const quic::QuicSocketAddress& server_address, + const quic::QuicSocketAddress& client_address, const std::string& hostname, + uint16_t signature_algorithm, quiche::QuicheStringPiece in, + std::unique_ptr callback) override; private: struct CertConfigWithFilterChain { diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_source_base.cc b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source_base.cc new file mode 100644 index 000000000000..220dc4cb1ccf --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source_base.cc @@ -0,0 +1,81 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_proof_source_base.h" + +#pragma GCC diagnostic push + +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include "quiche/quic/core/quic_data_writer.h" + +#pragma GCC diagnostic pop + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" + +namespace Envoy { +namespace Quic { + +void EnvoyQuicProofSourceBase::GetProof(const quic::QuicSocketAddress& server_address, + const quic::QuicSocketAddress& client_address, + const std::string& hostname, + const std::string& server_config, + quic::QuicTransportVersion /*transport_version*/, + quiche::QuicheStringPiece chlo_hash, + std::unique_ptr callback) { + quic::QuicReferenceCountedPointer chain = + GetCertChain(server_address, client_address, hostname); + + if (chain == nullptr || chain->certs.empty()) { + quic::QuicCryptoProof proof; + callback->Run(/*ok=*/false, nullptr, proof, nullptr); + return; + } + size_t payload_size = sizeof(quic::kProofSignatureLabel) + sizeof(uint32_t) + chlo_hash.size() + + server_config.size(); + auto payload = std::make_unique(payload_size); + quic::QuicDataWriter payload_writer(payload_size, payload.get(), + quiche::Endianness::HOST_BYTE_ORDER); + bool success = + payload_writer.WriteBytes(quic::kProofSignatureLabel, sizeof(quic::kProofSignatureLabel)) && + payload_writer.WriteUInt32(chlo_hash.size()) && payload_writer.WriteStringPiece(chlo_hash) && + payload_writer.WriteStringPiece(server_config); + if (!success) { + quic::QuicCryptoProof proof; + callback->Run(/*ok=*/false, nullptr, proof, nullptr); + return; + } + + std::string error_details; + bssl::UniquePtr cert = parseDERCertificate(chain->certs[0], &error_details); + if (cert == nullptr) { + ENVOY_LOG(warn, absl::StrCat("Invalid leaf cert: ", error_details)); + quic::QuicCryptoProof proof; + callback->Run(/*ok=*/false, nullptr, proof, nullptr); + return; + } + + bssl::UniquePtr pub_key(X509_get_pubkey(cert.get())); + int sign_alg = deduceSignatureAlgorithmFromPublicKey(pub_key.get(), &error_details); + if (sign_alg == 0) { + ENVOY_LOG(warn, absl::StrCat("Failed to deduce signature algorithm from public key: ", + error_details)); + quic::QuicCryptoProof proof; + callback->Run(/*ok=*/false, nullptr, proof, nullptr); + return; + } + + auto signature_callback = std::make_unique(std::move(callback), chain); + + signPayload(server_address, client_address, hostname, sign_alg, + quiche::QuicheStringPiece(payload.get(), payload_size), + std::move(signature_callback)); +} + +void EnvoyQuicProofSourceBase::ComputeTlsSignature( + const quic::QuicSocketAddress& server_address, const quic::QuicSocketAddress& client_address, + const std::string& hostname, uint16_t signature_algorithm, quiche::QuicheStringPiece in, + std::unique_ptr callback) { + signPayload(server_address, client_address, hostname, signature_algorithm, in, + std::move(callback)); +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source_base.h similarity index 68% rename from source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h rename to source/extensions/quic_listeners/quiche/envoy_quic_proof_source_base.h index f4a2a9466f42..149cc50c7d63 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source_base.h @@ -12,14 +12,16 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #include "quiche/quic/core/crypto/proof_source.h" #include "quiche/quic/core/quic_versions.h" - +#include "quiche/quic/core/crypto/crypto_protocol.h" +#include "quiche/quic/platform/api/quic_reference_counted.h" +#include "quiche/quic/platform/api/quic_socket_address.h" +#include "quiche/common/platform/api/quiche_string_piece.h" #pragma GCC diagnostic pop #include "openssl/ssl.h" #include "envoy/network/filter.h" -#include "quiche/quic/platform/api/quic_reference_counted.h" -#include "quiche/quic/platform/api/quic_socket_address.h" -#include "quiche/common/platform/api/quiche_string_piece.h" +#include "server/backtrace.h" +#include "common/common/logger.h" namespace Envoy { namespace Quic { @@ -38,11 +40,12 @@ class EnvoyQuicProofSourceDetails : public quic::ProofSource::Details { const Network::FilterChain& filter_chain_; }; -// A fake implementation of quic::ProofSource which uses RSA cipher suite to sign in GetProof(). -// TODO(danzh) Rename it to EnvoyQuicProofSource once it's fully implemented. -class EnvoyQuicFakeProofSource : public quic::ProofSource { +// A partial implementation of quic::ProofSource which chooses a cipher suite according to the leaf +// cert to sign in GetProof(). +class EnvoyQuicProofSourceBase : public quic::ProofSource, + protected Logger::Loggable { public: - ~EnvoyQuicFakeProofSource() override = default; + ~EnvoyQuicProofSourceBase() override = default; // quic::ProofSource // Returns a certs chain and its fake SCT "Fake timestamp" and TLS signature wrapped @@ -50,19 +53,24 @@ class EnvoyQuicFakeProofSource : public quic::ProofSource { void GetProof(const quic::QuicSocketAddress& server_address, const quic::QuicSocketAddress& client_address, const std::string& hostname, const std::string& server_config, quic::QuicTransportVersion /*transport_version*/, - quiche::QuicheStringPiece /*chlo_hash*/, - std::unique_ptr callback) override { - quic::QuicReferenceCountedPointer chain = - GetCertChain(server_address, client_address, hostname); - quic::QuicCryptoProof proof; - // TODO(danzh) Get the signature algorithm from leaf cert. - auto signature_callback = std::make_unique(std::move(callback), chain); - ComputeTlsSignature(server_address, client_address, hostname, SSL_SIGN_RSA_PSS_RSAE_SHA256, - server_config, std::move(signature_callback)); - } + quiche::QuicheStringPiece chlo_hash, + std::unique_ptr callback) override; TicketCrypter* GetTicketCrypter() override { return nullptr; } + void ComputeTlsSignature(const quic::QuicSocketAddress& server_address, + const quic::QuicSocketAddress& client_address, + const std::string& hostname, uint16_t signature_algorithm, + quiche::QuicheStringPiece in, + std::unique_ptr callback) override; + +protected: + virtual void signPayload(const quic::QuicSocketAddress& server_address, + const quic::QuicSocketAddress& client_address, + const std::string& hostname, uint16_t signature_algorithm, + quiche::QuicheStringPiece in, + std::unique_ptr callback) PURE; + private: // Used by GetProof() to get signature. class SignatureCallback : public quic::ProofSource::SignatureCallback { diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.cc b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.cc new file mode 100644 index 000000000000..b7040d1279d7 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.cc @@ -0,0 +1,48 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" + +#include "quiche/quic/core/crypto/certificate_view.h" + +namespace Envoy { +namespace Quic { + +quic::QuicAsyncStatus EnvoyQuicProofVerifier::VerifyCertChain( + const std::string& hostname, const uint16_t /*port*/, const std::vector& certs, + const std::string& /*ocsp_response*/, const std::string& /*cert_sct*/, + const quic::ProofVerifyContext* /*context*/, std::string* error_details, + std::unique_ptr* /*details*/, + std::unique_ptr /*callback*/) { + ASSERT(!certs.empty()); + bssl::UniquePtr intermediates(sk_X509_new_null()); + bssl::UniquePtr leaf; + for (size_t i = 0; i < certs.size(); i++) { + bssl::UniquePtr cert = parseDERCertificate(certs[i], error_details); + if (!cert) { + return quic::QUIC_FAILURE; + } + if (i == 0) { + leaf = std::move(cert); + } else { + sk_X509_push(intermediates.get(), cert.release()); + } + } + bool success = context_impl_.verifyCertChain(*leaf, *intermediates, *error_details); + if (!success) { + return quic::QUIC_FAILURE; + } + + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(certs[0]); + ASSERT(cert_view != nullptr); + for (const absl::string_view config_san : cert_view->subject_alt_name_domains()) { + if (Extensions::TransportSockets::Tls::ContextImpl::dnsNameMatch(hostname, config_san)) { + return quic::QUIC_SUCCESS; + } + } + *error_details = absl::StrCat("Leaf certificate doesn't match hostname: ", hostname); + return quic::QUIC_FAILURE; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h new file mode 100644 index 000000000000..a29eb999119f --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h @@ -0,0 +1,30 @@ +#pragma once + +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.h" +#include "extensions/transport_sockets/tls/context_impl.h" + +namespace Envoy { +namespace Quic { + +// A quic::ProofVerifier implementation which verifies cert chain using SSL +// client context config. +class EnvoyQuicProofVerifier : public EnvoyQuicProofVerifierBase { +public: + EnvoyQuicProofVerifier(Stats::Scope& scope, const Envoy::Ssl::ClientContextConfig& config, + TimeSource& time_source) + : context_impl_(scope, config, time_source) {} + + // EnvoyQuicProofVerifierBase + quic::QuicAsyncStatus + VerifyCertChain(const std::string& hostname, const uint16_t port, + const std::vector& certs, const std::string& ocsp_response, + const std::string& cert_sct, const quic::ProofVerifyContext* context, + std::string* error_details, std::unique_ptr* details, + std::unique_ptr callback) override; + +private: + Extensions::TransportSockets::Tls::ClientContextImpl context_impl_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.cc b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.cc new file mode 100644 index 000000000000..229b3ab36628 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.cc @@ -0,0 +1,70 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" + +#include "openssl/ssl.h" +#include "quiche/quic/core/crypto/certificate_view.h" +#include "quiche/quic/core/crypto/crypto_protocol.h" +#include "quiche/quic/core/quic_data_writer.h" + +namespace Envoy { +namespace Quic { + +quic::QuicAsyncStatus EnvoyQuicProofVerifierBase::VerifyProof( + const std::string& hostname, const uint16_t port, const std::string& server_config, + quic::QuicTransportVersion /*quic_version*/, absl::string_view chlo_hash, + const std::vector& certs, const std::string& cert_sct, + const std::string& signature, const quic::ProofVerifyContext* context, + std::string* error_details, std::unique_ptr* details, + std::unique_ptr callback) { + if (certs.empty()) { + *error_details = "Received empty cert chain."; + return quic::QUIC_FAILURE; + } + if (!verifySignature(server_config, chlo_hash, certs[0], signature, error_details)) { + return quic::QUIC_FAILURE; + } + + return VerifyCertChain(hostname, port, certs, "", cert_sct, context, error_details, details, + std::move(callback)); +} + +bool EnvoyQuicProofVerifierBase::verifySignature(const std::string& server_config, + absl::string_view chlo_hash, + const std::string& cert, + const std::string& signature, + std::string* error_details) { + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(cert); + if (cert_view == nullptr) { + *error_details = "Invalid leaf cert."; + return false; + } + int sign_alg = deduceSignatureAlgorithmFromPublicKey(cert_view->public_key(), error_details); + if (sign_alg == 0) { + return false; + } + + size_t payload_size = sizeof(quic::kProofSignatureLabel) + sizeof(uint32_t) + chlo_hash.size() + + server_config.size(); + auto payload = std::make_unique(payload_size); + quic::QuicDataWriter payload_writer(payload_size, payload.get(), + quiche::Endianness::HOST_BYTE_ORDER); + bool success = + payload_writer.WriteBytes(quic::kProofSignatureLabel, sizeof(quic::kProofSignatureLabel)) && + payload_writer.WriteUInt32(chlo_hash.size()) && payload_writer.WriteStringPiece(chlo_hash) && + payload_writer.WriteStringPiece(server_config); + if (!success) { + *error_details = "QuicPacketWriter error."; + return false; + } + bool valid = cert_view->VerifySignature(quiche::QuicheStringPiece(payload.get(), payload_size), + signature, sign_alg); + if (!valid) { + *error_details = "Signature is not valid."; + } + return valid; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.h b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.h new file mode 100644 index 000000000000..02dac5facd42 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.h @@ -0,0 +1,47 @@ +#pragma once + +#include "absl/strings/str_cat.h" + +#pragma GCC diagnostic push + +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" + +#include "quiche/quic/core/crypto/proof_verifier.h" +#include "quiche/quic/core/quic_versions.h" + +#pragma GCC diagnostic pop + +#include "common/common/logger.h" + +namespace Envoy { +namespace Quic { + +// A partial implementation of quic::ProofVerifier which does signature +// verification. +class EnvoyQuicProofVerifierBase : public quic::ProofVerifier, + protected Logger::Loggable { +public: + ~EnvoyQuicProofVerifierBase() override = default; + + // quic::ProofVerifier + // Return success if the certs chain is valid and signature of { + // server_config + chlo_hash} is valid. Otherwise failure. + quic::QuicAsyncStatus + VerifyProof(const std::string& hostname, const uint16_t port, const std::string& server_config, + quic::QuicTransportVersion /*quic_version*/, absl::string_view chlo_hash, + const std::vector& certs, const std::string& cert_sct, + const std::string& signature, const quic::ProofVerifyContext* context, + std::string* error_details, std::unique_ptr* details, + std::unique_ptr callback) override; + + std::unique_ptr CreateDefaultContext() override { return nullptr; } + +protected: + virtual bool verifySignature(const std::string& server_config, absl::string_view chlo_hash, + const std::string& cert, const std::string& signature, + std::string* error_details); +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc b/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc index aefb6a860e5e..b5c710a81269 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc @@ -124,5 +124,66 @@ createConnectionSocket(Network::Address::InstanceConstSharedPtr& peer_addr, return connection_socket; } +bssl::UniquePtr parseDERCertificate(const std::string& der_bytes, + std::string* error_details) { + const uint8_t* data; + const uint8_t* orig_data; + orig_data = data = reinterpret_cast(der_bytes.data()); + bssl::UniquePtr cert(d2i_X509(nullptr, &data, der_bytes.size())); + if (!cert.get()) { + *error_details = "d2i_X509: fail to parse DER"; + return nullptr; + } + if (data < orig_data || static_cast(data - orig_data) != der_bytes.size()) { + *error_details = "There is trailing garbage in DER."; + return nullptr; + } + return cert; +} + +int deduceSignatureAlgorithmFromPublicKey(const EVP_PKEY* public_key, std::string* error_details) { + int sign_alg = 0; + const int pkey_id = EVP_PKEY_id(public_key); + switch (pkey_id) { + case EVP_PKEY_EC: { + // We only support P-256 ECDSA today. + const EC_KEY* ecdsa_public_key = EVP_PKEY_get0_EC_KEY(public_key); + // Since we checked the key type above, this should be valid. + ASSERT(ecdsa_public_key != nullptr); + const EC_GROUP* ecdsa_group = EC_KEY_get0_group(ecdsa_public_key); + if (ecdsa_group == nullptr || EC_GROUP_get_curve_name(ecdsa_group) != NID_X9_62_prime256v1) { + *error_details = "Invalid leaf cert, only P-256 ECDSA certificates are supported"; + break; + } + // QUICHE uses SHA-256 as hash function in cert signature. + sign_alg = SSL_SIGN_ECDSA_SECP256R1_SHA256; + } break; + case EVP_PKEY_RSA: { + // We require RSA certificates with 2048-bit or larger keys. + const RSA* rsa_public_key = EVP_PKEY_get0_RSA(public_key); + // Since we checked the key type above, this should be valid. + ASSERT(rsa_public_key != nullptr); + const unsigned rsa_key_length = RSA_size(rsa_public_key); +#ifdef BORINGSSL_FIPS + if (rsa_key_length != 2048 / 8 && rsa_key_length != 3072 / 8) { + *error_details = "Invalid leaf cert, only RSA certificates with 2048-bit or 3072-bit keys " + "are supported in FIPS mode"; + break; + } +#else + if (rsa_key_length < 2048 / 8) { + *error_details = + "Invalid leaf cert, only RSA certificates with 2048-bit or larger keys are supported"; + break; + } +#endif + sign_alg = SSL_SIGN_RSA_PSS_RSAE_SHA256; + } break; + default: + *error_details = "Invalid leaf cert, only RSA and ECDSA certificates are supported"; + } + return sign_alg; +} + } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_utils.h b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h index f5714ef15b83..34dce87d836b 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_utils.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h @@ -24,6 +24,8 @@ #include "quiche/quic/platform/api/quic_ip_address.h" #include "quiche/quic/platform/api/quic_socket_address.h" +#include "openssl/ssl.h" + namespace Envoy { namespace Quic { @@ -80,5 +82,14 @@ createConnectionSocket(Network::Address::InstanceConstSharedPtr& peer_addr, Network::Address::InstanceConstSharedPtr& local_addr, const Network::ConnectionSocket::OptionsSharedPtr& options); +// Convert a cert in string form to X509 object. +// Return nullptr if the bytes passed cannot be passed. +bssl::UniquePtr parseDERCertificate(const std::string& der_bytes, std::string* error_details); + +// Deduce the suitable signature algorithm according to the public key. +// Return the sign algorithm id works with the public key; If the public key is +// not supported, return 0 with error_details populated correspondingly. +int deduceSignatureAlgorithmFromPublicKey(const EVP_PKEY* public_key, std::string* error_details); + } // namespace Quic } // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/context_config_impl.h b/source/extensions/transport_sockets/tls/context_config_impl.h index 9cfaff0482fb..ad2d927d8231 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.h +++ b/source/extensions/transport_sockets/tls/context_config_impl.h @@ -98,6 +98,9 @@ class ContextConfigImpl : public virtual Ssl::ContextConfig { class ClientContextConfigImpl : public ContextConfigImpl, public Envoy::Ssl::ClientContextConfig { public: + static const std::string DEFAULT_CIPHER_SUITES; + static const std::string DEFAULT_CURVES; + ClientContextConfigImpl( const envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext& config, absl::string_view sigalgs, @@ -116,8 +119,6 @@ class ClientContextConfigImpl : public ContextConfigImpl, public Envoy::Ssl::Cli private: static const unsigned DEFAULT_MIN_VERSION; static const unsigned DEFAULT_MAX_VERSION; - static const std::string DEFAULT_CIPHER_SUITES; - static const std::string DEFAULT_CURVES; const std::string server_name_indication_; const bool allow_renegotiation_; diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index 369bdd460f98..502739958e50 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -527,49 +527,50 @@ int ContextImpl::verifyCallback(X509_STORE_CTX* store_ctx, void* arg) { ContextImpl* impl = reinterpret_cast(arg); SSL* ssl = reinterpret_cast( X509_STORE_CTX_get_ex_data(store_ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); - Envoy::Ssl::SslExtendedSocketInfo* sslExtendedInfo = + auto cert = bssl::UniquePtr(SSL_get_peer_certificate(ssl)); + return impl->doVerifyCertChain( + store_ctx, reinterpret_cast( - SSL_get_ex_data(ssl, ContextImpl::sslExtendedSocketInfoIndex())); + SSL_get_ex_data(ssl, ContextImpl::sslExtendedSocketInfoIndex())), + *cert, static_cast(SSL_get_app_data(ssl))); +} - if (impl->verify_trusted_ca_) { +int ContextImpl::doVerifyCertChain( + X509_STORE_CTX* store_ctx, Ssl::SslExtendedSocketInfo* ssl_extended_info, X509& leaf_cert, + const Network::TransportSocketOptions* transport_socket_options) { + if (verify_trusted_ca_) { int ret = X509_verify_cert(store_ctx); - if (sslExtendedInfo) { - sslExtendedInfo->setCertificateValidationStatus( + if (ssl_extended_info) { + ssl_extended_info->setCertificateValidationStatus( ret == 1 ? Envoy::Ssl::ClientValidationStatus::Validated : Envoy::Ssl::ClientValidationStatus::Failed); } if (ret <= 0) { - impl->stats_.fail_verify_error_.inc(); - return impl->allow_untrusted_certificate_ ? 1 : ret; + stats_.fail_verify_error_.inc(); + return allow_untrusted_certificate_ ? 1 : ret; } } - bssl::UniquePtr cert(SSL_get_peer_certificate(ssl)); - - const Network::TransportSocketOptions* transport_socket_options = - static_cast(SSL_get_app_data(ssl)); - - Envoy::Ssl::ClientValidationStatus validated = impl->verifyCertificate( - cert.get(), + Envoy::Ssl::ClientValidationStatus validated = verifyCertificate( + &leaf_cert, transport_socket_options && !transport_socket_options->verifySubjectAltNameListOverride().empty() ? transport_socket_options->verifySubjectAltNameListOverride() - : impl->verify_subject_alt_name_list_, - impl->subject_alt_name_matchers_); + : verify_subject_alt_name_list_, + subject_alt_name_matchers_); - if (sslExtendedInfo) { - if (sslExtendedInfo->certificateValidationStatus() == + if (ssl_extended_info) { + if (ssl_extended_info->certificateValidationStatus() == Envoy::Ssl::ClientValidationStatus::NotValidated) { - sslExtendedInfo->setCertificateValidationStatus(validated); + ssl_extended_info->setCertificateValidationStatus(validated); } else if (validated != Envoy::Ssl::ClientValidationStatus::NotValidated) { - sslExtendedInfo->setCertificateValidationStatus(validated); + ssl_extended_info->setCertificateValidationStatus(validated); } } - return impl->allow_untrusted_certificate_ - ? 1 - : (validated != Envoy::Ssl::ClientValidationStatus::Failed); + return allow_untrusted_certificate_ ? 1 + : (validated != Envoy::Ssl::ClientValidationStatus::Failed); } Envoy::Ssl::ClientValidationStatus ContextImpl::verifyCertificate( @@ -675,7 +676,7 @@ bool ContextImpl::matchSubjectAltName( if (general_name->type == GEN_DNS && config_san_matcher.matcher().match_pattern_case() == envoy::type::matcher::v3::StringMatcher::MatchPatternCase::kExact - ? dnsNameMatch(config_san_matcher.matcher().exact(), san.c_str()) + ? dnsNameMatch(config_san_matcher.matcher().exact(), absl::string_view(san)) : config_san_matcher.match(san)) { return true; } @@ -703,20 +704,20 @@ bool ContextImpl::verifySubjectAltName(X509* cert, return false; } -bool ContextImpl::dnsNameMatch(const std::string& dns_name, const char* pattern) { +bool ContextImpl::dnsNameMatch(const absl::string_view dns_name, const absl::string_view pattern) { if (dns_name == pattern) { return true; } - size_t pattern_len = strlen(pattern); + size_t pattern_len = pattern.length(); if (pattern_len > 1 && pattern[0] == '*' && pattern[1] == '.') { if (dns_name.length() > pattern_len - 1) { const size_t off = dns_name.length() - pattern_len + 1; if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.fix_wildcard_matching")) { return dns_name.substr(0, off).find('.') == std::string::npos && - dns_name.compare(off, pattern_len - 1, pattern + 1) == 0; + dns_name.substr(off, pattern_len - 1) == pattern.substr(1, pattern_len - 1); } else { - return dns_name.compare(off, pattern_len - 1, pattern + 1) == 0; + return dns_name.substr(off, pattern_len - 1) == pattern.substr(1, pattern_len - 1); } } } @@ -1394,6 +1395,28 @@ bool ServerContextImpl::TlsContext::isCipherEnabled(uint16_t cipher_id, uint16_t return false; } +bool ContextImpl::verifyCertChain(X509& leaf_cert, STACK_OF(X509) & intermediates, + std::string& error_details) { + bssl::UniquePtr ctx(X509_STORE_CTX_new()); + // It doesn't matter which SSL context is used, because they share the same + // cert validation config. + X509_STORE* store = SSL_CTX_get_cert_store(tls_contexts_[0].ssl_ctx_.get()); + if (!X509_STORE_CTX_init(ctx.get(), store, &leaf_cert, &intermediates)) { + error_details = "Failed to verify certificate chain: X509_STORE_CTX_init"; + return false; + } + + int res = doVerifyCertChain(ctx.get(), nullptr, leaf_cert, nullptr); + if (res <= 0) { + const int n = X509_STORE_CTX_get_error(ctx.get()); + const int depth = X509_STORE_CTX_get_error_depth(ctx.get()); + error_details = absl::StrCat("X509_verify_cert: certificate verification error at depth ", + depth, ": ", X509_verify_cert_error_string(n)); + return false; + } + return true; +} + } // namespace Tls } // namespace TransportSockets } // namespace Extensions diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index 407dd45f86f8..5ea35a48228e 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -84,7 +84,7 @@ class ContextImpl : public virtual Envoy::Ssl::Context { * @param pattern the pattern to match against (*.example.com) * @return true if the san matches pattern */ - static bool dnsNameMatch(const std::string& dns_name, const char* pattern); + static bool dnsNameMatch(const absl::string_view dns_name, const absl::string_view pattern); SslStats& stats() { return stats_; } @@ -101,6 +101,8 @@ class ContextImpl : public virtual Envoy::Ssl::Context { std::vector getPrivateKeyMethodProviders(); + bool verifyCertChain(X509& leaf_cert, STACK_OF(X509) & intermediates, std::string& error_details); + protected: ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& config, TimeSource& time_source); @@ -117,6 +119,11 @@ class ContextImpl : public virtual Envoy::Ssl::Context { // A SSL_CTX_set_cert_verify_callback for custom cert validation. static int verifyCallback(X509_STORE_CTX* store_ctx, void* arg); + // Called by verifyCallback to do the actual cert chain verification. + int doVerifyCertChain(X509_STORE_CTX* store_ctx, Ssl::SslExtendedSocketInfo* ssl_extended_info, + X509& leaf_cert, + const Network::TransportSocketOptions* transport_socket_options); + Envoy::Ssl::ClientValidationStatus verifyCertificate(X509* cert, const std::vector& verify_san_list, const std::vector& subject_alt_name_matchers); diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index b3c4eb70698d..29ae0a89eb28 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -49,6 +49,7 @@ envoy_cc_test( deps = [ "//source/extensions/quic_listeners/quiche:envoy_quic_proof_source_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_proof_verifier_lib", + "//source/extensions/transport_sockets/tls:context_config_lib", "//test/mocks/network:network_mocks", "//test/mocks/ssl:ssl_mocks", "@com_googlesource_quiche//:quic_core_versions_lib", @@ -56,6 +57,19 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "envoy_quic_proof_verifier_test", + srcs = ["envoy_quic_proof_verifier_test.cc"], + external_deps = ["quiche_quic_platform"], + tags = ["nofips"], + deps = [ + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_verifier_lib", + "//source/extensions/transport_sockets/tls:context_config_lib", + "//test/mocks/ssl:ssl_mocks", + "@com_googlesource_quiche//:quic_test_tools_test_certificates_lib", + ], +) + envoy_cc_test( name = "envoy_quic_server_stream_test", srcs = ["envoy_quic_server_stream_test.cc"], @@ -221,19 +235,28 @@ envoy_cc_test_library( hdrs = ["test_proof_source.h"], tags = ["nofips"], deps = [ - "//source/extensions/quic_listeners/quiche:envoy_quic_fake_proof_source_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_source_base_lib", "//test/mocks/network:network_mocks", "@com_googlesource_quiche//:quic_test_tools_test_certificates_lib", ], ) +envoy_cc_test_library( + name = "test_proof_verifier_lib", + hdrs = ["test_proof_verifier.h"], + tags = ["nofips"], + deps = [ + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_verifier_base_lib", + ], +) + envoy_cc_test_library( name = "quic_test_utils_for_envoy_lib", srcs = ["crypto_test_utils_for_envoy.cc"], tags = ["nofips"], deps = [ ":test_proof_source_lib", - "//source/extensions/quic_listeners/quiche:envoy_quic_proof_verifier_lib", + ":test_proof_verifier_lib", "@com_googlesource_quiche//:quic_test_tools_test_utils_interface_lib", ], ) diff --git a/test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc b/test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc index c5b7a11d70e3..cafdce0c6227 100644 --- a/test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc +++ b/test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc @@ -19,7 +19,7 @@ #endif #include -#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h" +#include "test/extensions/quic_listeners/quiche/test_proof_verifier.h" #include "test/extensions/quic_listeners/quiche/test_proof_source.h" namespace quic { @@ -32,7 +32,7 @@ std::unique_ptr ProofSourceForTesting() { // NOLINTNEXTLINE(readability-identifier-naming) std::unique_ptr ProofVerifierForTesting() { - return std::make_unique(); + return std::make_unique(); } // NOLINTNEXTLINE(readability-identifier-naming) diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc index e61e34eac270..d896dbb86b7c 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc @@ -2,9 +2,10 @@ #include #include -#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h" #include "extensions/quic_listeners/quiche/envoy_quic_proof_source.h" +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/transport_sockets/tls/context_config_impl.h" #include "test/mocks/network/mocks.h" #include "test/mocks/ssl/mocks.h" @@ -23,29 +24,105 @@ namespace Quic { class TestGetProofCallback : public quic::ProofSource::Callback { public: - TestGetProofCallback(bool& called, std::string leaf_cert_scts, const absl::string_view cert, + TestGetProofCallback(bool& called, bool should_succeed, const std::string& server_config, + quic::QuicTransportVersion& version, quiche::QuicheStringPiece chlo_hash, Network::FilterChain& filter_chain) - : called_(called), expected_leaf_certs_scts_(std::move(leaf_cert_scts)), - expected_leaf_cert_(cert), expected_filter_chain_(filter_chain) {} + : called_(called), should_succeed_(should_succeed), server_config_(server_config), + version_(version), chlo_hash_(chlo_hash), expected_filter_chain_(filter_chain) { + ON_CALL(client_context_config_, cipherSuites) + .WillByDefault(ReturnRef( + Extensions::TransportSockets::Tls::ClientContextConfigImpl::DEFAULT_CIPHER_SUITES)); + ON_CALL(client_context_config_, ecdhCurves) + .WillByDefault( + ReturnRef(Extensions::TransportSockets::Tls::ClientContextConfigImpl::DEFAULT_CURVES)); + const std::string alpn("h2,http/1.1"); + ON_CALL(client_context_config_, alpnProtocols()).WillByDefault(ReturnRef(alpn)); + const std::string empty_string; + ON_CALL(client_context_config_, serverNameIndication()).WillByDefault(ReturnRef(empty_string)); + ON_CALL(client_context_config_, signingAlgorithmsForTest()) + .WillByDefault(ReturnRef(empty_string)); + ON_CALL(client_context_config_, certificateValidationContext()) + .WillByDefault(Return(&cert_validation_ctx_config_)); + + // Getting the last cert in the chain as the root CA cert. + std::string cert_chain(quic::test::kTestCertificateChainPem); + const std::string& root_ca_cert = + cert_chain.substr(cert_chain.rfind("-----BEGIN CERTIFICATE-----")); + const std::string path_string("some_path"); + ON_CALL(cert_validation_ctx_config_, caCert()).WillByDefault(ReturnRef(root_ca_cert)); + ON_CALL(cert_validation_ctx_config_, caCertPath()).WillByDefault(ReturnRef(path_string)); + ON_CALL(cert_validation_ctx_config_, trustChainVerification) + .WillByDefault(Return(envoy::extensions::transport_sockets::tls::v3:: + CertificateValidationContext::VERIFY_TRUST_CHAIN)); + ON_CALL(cert_validation_ctx_config_, allowExpiredCertificate()).WillByDefault(Return(true)); + const std::string crl_list; + ON_CALL(cert_validation_ctx_config_, certificateRevocationList()) + .WillByDefault(ReturnRef(crl_list)); + ON_CALL(cert_validation_ctx_config_, certificateRevocationListPath()) + .WillByDefault(ReturnRef(path_string)); + const std::vector empty_string_list; + ON_CALL(cert_validation_ctx_config_, verifySubjectAltNameList()) + .WillByDefault(ReturnRef(empty_string_list)); + const std::vector san_matchers; + ON_CALL(cert_validation_ctx_config_, subjectAltNameMatchers()) + .WillByDefault(ReturnRef(san_matchers)); + ON_CALL(cert_validation_ctx_config_, verifyCertificateHashList()) + .WillByDefault(ReturnRef(empty_string_list)); + ON_CALL(cert_validation_ctx_config_, verifyCertificateSpkiList()) + .WillByDefault(ReturnRef(empty_string_list)); + verifier_ = + std::make_unique(store_, client_context_config_, time_system_); + } // quic::ProofSource::Callback void Run(bool ok, const quic::QuicReferenceCountedPointer& chain, const quic::QuicCryptoProof& proof, std::unique_ptr details) override { + called_ = true; + if (!should_succeed_) { + EXPECT_FALSE(ok); + return; + }; EXPECT_TRUE(ok); - EXPECT_EQ(expected_leaf_certs_scts_, proof.leaf_cert_scts); EXPECT_EQ(2, chain->certs.size()); - EXPECT_EQ(expected_leaf_cert_, chain->certs[0]); + std::string error; + EXPECT_EQ(quic::QUIC_SUCCESS, + verifier_->VerifyProof("www.example.org", 54321, server_config_, version_, chlo_hash_, + chain->certs, proof.leaf_cert_scts, proof.signature, nullptr, + &error, nullptr, nullptr)) + << error; EXPECT_EQ(&expected_filter_chain_, &static_cast(details.get())->filterChain()); - called_ = true; } private: bool& called_; - std::string expected_leaf_certs_scts_; - absl::string_view expected_leaf_cert_; + bool should_succeed_; + const std::string& server_config_; + const quic::QuicTransportVersion& version_; + quiche::QuicheStringPiece chlo_hash_; Network::FilterChain& expected_filter_chain_; + NiceMock store_; + Event::GlobalTimeSystem time_system_; + NiceMock client_context_config_; + NiceMock cert_validation_ctx_config_; + std::unique_ptr verifier_; +}; + +class TestSignatureCallback : public quic::ProofSource::SignatureCallback { +public: + TestSignatureCallback(bool expect_success) : expect_success_(expect_success) {} + ~TestSignatureCallback() override { EXPECT_TRUE(run_called_); } + + // quic::ProofSource::SignatureCallback + void Run(bool ok, std::string, std::unique_ptr) override { + EXPECT_EQ(expect_success_, ok); + run_called_ = true; + } + +private: + bool expect_success_; + bool run_called_{false}; }; class EnvoyQuicProofSourceTest : public ::testing::Test { @@ -53,17 +130,55 @@ class EnvoyQuicProofSourceTest : public ::testing::Test { EnvoyQuicProofSourceTest() : server_address_(quic::QuicIpAddress::Loopback4(), 12345), client_address_(quic::QuicIpAddress::Loopback4(), 54321), + transport_socket_factory_(std::make_unique()), listener_stats_({ALL_LISTENER_STATS(POOL_COUNTER(listener_config_.listenerScope()), POOL_GAUGE(listener_config_.listenerScope()), POOL_HISTOGRAM(listener_config_.listenerScope()))}), proof_source_(listen_socket_, filter_chain_manager_, listener_stats_) {} + void expectCertChainAndPrivateKey(const std::string& cert, bool expect_private_key) { + EXPECT_CALL(listen_socket_, ioHandle()).Times(expect_private_key ? 2u : 1u); + EXPECT_CALL(filter_chain_manager_, findFilterChain(_)) + .WillRepeatedly(Invoke([&](const Network::ConnectionSocket& connection_socket) { + EXPECT_EQ(*quicAddressToEnvoyAddressInstance(server_address_), + *connection_socket.localAddress()); + EXPECT_EQ(*quicAddressToEnvoyAddressInstance(client_address_), + *connection_socket.remoteAddress()); + EXPECT_EQ(Extensions::TransportSockets::TransportProtocolNames::get().Quic, + connection_socket.detectedTransportProtocol()); + EXPECT_EQ("h2", connection_socket.requestedApplicationProtocols()[0]); + return &filter_chain_; + })); + EXPECT_CALL(filter_chain_, transportSocketFactory()) + .WillRepeatedly(ReturnRef(transport_socket_factory_)); + + std::vector> tls_cert_configs{ + std::reference_wrapper(tls_cert_config_)}; + EXPECT_CALL(dynamic_cast( + transport_socket_factory_.serverContextConfig()), + tlsCertificates()) + .WillRepeatedly(Return(tls_cert_configs)); + EXPECT_CALL(tls_cert_config_, certificateChain()).WillOnce(ReturnRef(cert)); + if (expect_private_key) { + EXPECT_CALL(tls_cert_config_, privateKey()).WillOnce(ReturnRef(pkey_)); + } + } + + void testGetProof(bool expect_success) { + bool called = false; + auto callback = std::make_unique(called, expect_success, server_config_, + version_, chlo_hash_, filter_chain_); + proof_source_.GetProof(server_address_, client_address_, hostname_, server_config_, version_, + chlo_hash_, std::move(callback)); + EXPECT_TRUE(called); + } + protected: std::string hostname_{"www.fake.com"}; quic::QuicSocketAddress server_address_; quic::QuicSocketAddress client_address_; quic::QuicTransportVersion version_{quic::QUIC_VERSION_UNSUPPORTED}; - quiche::QuicheStringPiece chlo_hash_{""}; + quiche::QuicheStringPiece chlo_hash_{"aaaaa"}; std::string server_config_{"Server Config"}; std::string expected_certs_{quic::test::kTestCertificateChainPem}; std::string pkey_{quic::test::kTestCertificatePrivateKeyPem}; @@ -71,27 +186,66 @@ class EnvoyQuicProofSourceTest : public ::testing::Test { Network::MockFilterChainManager filter_chain_manager_; Network::MockListenSocket listen_socket_; testing::NiceMock listener_config_; + QuicServerTransportSocketFactory transport_socket_factory_; + Ssl::MockTlsCertificateConfig tls_cert_config_; Server::ListenerStats listener_stats_; EnvoyQuicProofSource proof_source_; - EnvoyQuicFakeProofVerifier proof_verifier_; }; TEST_F(EnvoyQuicProofSourceTest, TestGetProof) { + expectCertChainAndPrivateKey(expected_certs_, true); + testGetProof(true); +} + +TEST_F(EnvoyQuicProofSourceTest, GetProofFailNoFilterChain) { bool called = false; - auto callback = std::make_unique( - called, "Fake timestamp", quic::test::kTestCertificate, filter_chain_); - EXPECT_CALL(listen_socket_, ioHandle()).Times(2); + auto callback = std::make_unique(called, false, server_config_, version_, + chlo_hash_, filter_chain_); + EXPECT_CALL(listen_socket_, ioHandle()); + EXPECT_CALL(filter_chain_manager_, findFilterChain(_)) + .WillRepeatedly(Invoke([&](const Network::ConnectionSocket&) { return nullptr; })); + proof_source_.GetProof(server_address_, client_address_, hostname_, server_config_, version_, + chlo_hash_, std::move(callback)); + EXPECT_TRUE(called); +} + +TEST_F(EnvoyQuicProofSourceTest, GetProofFailInvalidCert) { + std::string invalid_cert{R"(-----BEGIN CERTIFICATE----- + invalid certificate + -----END CERTIFICATE-----)"}; + expectCertChainAndPrivateKey(invalid_cert, false); + testGetProof(false); +} + +TEST_F(EnvoyQuicProofSourceTest, GetProofFailInvalidPublicKeyInCert) { + // This is a valid cert with RSA public key. But we don't support RSA key with + // length < 1024. + std::string cert_with_rsa_1024{R"(-----BEGIN CERTIFICATE----- +MIIC2jCCAkOgAwIBAgIUDBHEwlCvLGh3w0O8VwIW+CjYXY8wDQYJKoZIhvcNAQEL +BQAwfzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk1BMRIwEAYDVQQHDAlDYW1icmlk +Z2UxDzANBgNVBAoMBkdvb2dsZTEOMAwGA1UECwwFZW52b3kxDTALBgNVBAMMBHRl +c3QxHzAdBgkqhkiG9w0BCQEWEGRhbnpoQGdvb2dsZS5jb20wHhcNMjAwODA0MTg1 +OTQ4WhcNMjEwODA0MTg1OTQ4WjB/MQswCQYDVQQGEwJVUzELMAkGA1UECAwCTUEx +EjAQBgNVBAcMCUNhbWJyaWRnZTEPMA0GA1UECgwGR29vZ2xlMQ4wDAYDVQQLDAVl +bnZveTENMAsGA1UEAwwEdGVzdDEfMB0GCSqGSIb3DQEJARYQZGFuemhAZ29vZ2xl +LmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAykCZNjxws+sNfnp18nsp ++7LN81J/RSwAHLkGnwEtd3OxSUuiCYHgYlyuEAwJdf99+SaFrgcA4LvYJ/Mhm/fZ +msnpfsAvoQ49+ax0fm1x56ii4KgNiu9iFsWwwVmkHkgjlRcRsmhr4WeIf14Yvpqs +JNsbNVSCZ4GLQ2V6BqIHlhcCAwEAAaNTMFEwHQYDVR0OBBYEFDO1KPYcdRmeKDvL +H2Yzj8el2Xe1MB8GA1UdIwQYMBaAFDO1KPYcdRmeKDvLH2Yzj8el2Xe1MA8GA1Ud +EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADgYEAnwWVmwSK9TDml7oHGBavzOC1 +f/lOd5zz2e7Tu2pUtx1sX1tlKph1D0ANpJwxRV78R2hjmynLSl7h4Ual9NMubqkD +x96rVeUbRJ/qU4//nNM/XQa9vIAIcTZ0jFhmb0c3R4rmoqqC3vkSDwtaE5yuS5T4 +GUy+n0vQNB0cXGzgcGI= +-----END CERTIFICATE-----)"}; + expectCertChainAndPrivateKey(cert_with_rsa_1024, false); + testGetProof(false); +} + +TEST_F(EnvoyQuicProofSourceTest, InvalidPrivateKey) { + EXPECT_CALL(listen_socket_, ioHandle()); EXPECT_CALL(filter_chain_manager_, findFilterChain(_)) - .WillRepeatedly(Invoke([&](const Network::ConnectionSocket& connection_socket) { - EXPECT_EQ(*quicAddressToEnvoyAddressInstance(server_address_), - *connection_socket.localAddress()); - EXPECT_EQ(*quicAddressToEnvoyAddressInstance(client_address_), - *connection_socket.remoteAddress()); - EXPECT_EQ(Extensions::TransportSockets::TransportProtocolNames::get().Quic, - connection_socket.detectedTransportProtocol()); - EXPECT_EQ("h2", connection_socket.requestedApplicationProtocols()[0]); - return &filter_chain_; - })); + .WillOnce(Invoke([&](const Network::ConnectionSocket&) { return &filter_chain_; })); auto server_context_config = std::make_unique(); auto server_context_config_ptr = server_context_config.get(); QuicServerTransportSocketFactory transport_socket_factory(std::move(server_context_config)); @@ -103,20 +257,11 @@ TEST_F(EnvoyQuicProofSourceTest, TestGetProof) { std::reference_wrapper(tls_cert_config)}; EXPECT_CALL(*server_context_config_ptr, tlsCertificates()) .WillRepeatedly(Return(tls_cert_configs)); - EXPECT_CALL(tls_cert_config, certificateChain()).WillOnce(ReturnRef(expected_certs_)); - EXPECT_CALL(tls_cert_config, privateKey()).WillOnce(ReturnRef(pkey_)); - proof_source_.GetProof(server_address_, client_address_, hostname_, server_config_, version_, - chlo_hash_, std::move(callback)); - EXPECT_TRUE(called); - - EXPECT_EQ(quic::QUIC_SUCCESS, - proof_verifier_.VerifyProof(hostname_, /*port=*/0, server_config_, version_, chlo_hash_, - {"Fake cert"}, "", "fake signature", nullptr, nullptr, - nullptr, nullptr)); - EXPECT_EQ(quic::QUIC_FAILURE, - proof_verifier_.VerifyProof(hostname_, /*port=*/0, server_config_, version_, chlo_hash_, - {"Fake cert", "Unexpected cert"}, "Fake timestamp", - "fake signature", nullptr, nullptr, nullptr, nullptr)); + std::string invalid_pkey("abcdefg"); + EXPECT_CALL(tls_cert_config, privateKey()).WillOnce(ReturnRef(invalid_pkey)); + proof_source_.ComputeTlsSignature(server_address_, client_address_, hostname_, + SSL_SIGN_RSA_PSS_RSAE_SHA256, "payload", + std::make_unique(false)); } } // namespace Quic diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_test.cc new file mode 100644 index 000000000000..4a1dfe144dd3 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_proof_verifier_test.cc @@ -0,0 +1,252 @@ +#include +#include + +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" +#include "extensions/transport_sockets/tls/context_config_impl.h" + +#include "test/mocks/ssl/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/test_common/test_time.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "quiche/quic/core/crypto/certificate_view.h" +#include "quiche/quic/test_tools/test_certificates.h" + +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Quic { + +class EnvoyQuicProofVerifierTest : public testing::Test { +public: + EnvoyQuicProofVerifierTest() + : root_ca_cert_(cert_chain_.substr(cert_chain_.rfind("-----BEGIN CERTIFICATE-----"))), + leaf_cert_([=]() { + std::stringstream pem_stream(cert_chain_); + std::vector chain = quic::CertificateView::LoadPemFromStream(&pem_stream); + return chain[0]; + }()) { + ON_CALL(client_context_config_, cipherSuites) + .WillByDefault(ReturnRef( + Extensions::TransportSockets::Tls::ClientContextConfigImpl::DEFAULT_CIPHER_SUITES)); + ON_CALL(client_context_config_, ecdhCurves) + .WillByDefault( + ReturnRef(Extensions::TransportSockets::Tls::ClientContextConfigImpl::DEFAULT_CURVES)); + ON_CALL(client_context_config_, alpnProtocols()).WillByDefault(ReturnRef(alpn_)); + ON_CALL(client_context_config_, serverNameIndication()).WillByDefault(ReturnRef(empty_string_)); + ON_CALL(client_context_config_, signingAlgorithmsForTest()).WillByDefault(ReturnRef(sig_algs_)); + ON_CALL(client_context_config_, certificateValidationContext()) + .WillByDefault(Return(&cert_validation_ctx_config_)); + } + + // Since this cert chain contains an expired cert, we can flip allow_expired_cert to test the code + // paths for BoringSSL cert verification success and failure. + void configCertVerificationDetails(bool allow_expired_cert) { + // Getting the last cert in the chain as the root CA cert. + EXPECT_CALL(cert_validation_ctx_config_, caCert()).WillRepeatedly(ReturnRef(root_ca_cert_)); + EXPECT_CALL(cert_validation_ctx_config_, caCertPath()).WillRepeatedly(ReturnRef(path_string_)); + EXPECT_CALL(cert_validation_ctx_config_, trustChainVerification) + .WillRepeatedly(Return(envoy::extensions::transport_sockets::tls::v3:: + CertificateValidationContext::VERIFY_TRUST_CHAIN)); + EXPECT_CALL(cert_validation_ctx_config_, allowExpiredCertificate()) + .WillRepeatedly(Return(allow_expired_cert)); + EXPECT_CALL(cert_validation_ctx_config_, certificateRevocationList()) + .WillRepeatedly(ReturnRef(empty_string_)); + EXPECT_CALL(cert_validation_ctx_config_, certificateRevocationListPath()) + .WillRepeatedly(ReturnRef(path_string_)); + EXPECT_CALL(cert_validation_ctx_config_, verifySubjectAltNameList()) + .WillRepeatedly(ReturnRef(empty_string_list_)); + EXPECT_CALL(cert_validation_ctx_config_, subjectAltNameMatchers()) + .WillRepeatedly(ReturnRef(san_matchers_)); + EXPECT_CALL(cert_validation_ctx_config_, verifyCertificateHashList()) + .WillRepeatedly(ReturnRef(empty_string_list_)); + EXPECT_CALL(cert_validation_ctx_config_, verifyCertificateSpkiList()) + .WillRepeatedly(ReturnRef(empty_string_list_)); + verifier_ = + std::make_unique(store_, client_context_config_, time_system_); + } + +protected: + const std::string path_string_{"some_path"}; + const std::string alpn_{"h2,http/1.1"}; + const std::string sig_algs_{"rsa_pss_rsae_sha256"}; + const std::vector san_matchers_; + const std::string empty_string_; + const std::vector empty_string_list_; + const std::string cert_chain_{quic::test::kTestCertificateChainPem}; + const std::string root_ca_cert_; + const std::string leaf_cert_; + NiceMock store_; + Event::GlobalTimeSystem time_system_; + NiceMock client_context_config_; + Ssl::MockCertificateValidationContextConfig cert_validation_ctx_config_; + std::unique_ptr verifier_; +}; + +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainSuccess) { + configCertVerificationDetails(true); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + EXPECT_EQ(quic::QUIC_SUCCESS, + verifier_->VerifyCertChain(std::string(cert_view->subject_alt_name_domains()[0]), 54321, + {leaf_cert_}, ocsp_response, cert_sct, nullptr, + &error_details, nullptr, nullptr)) + << error_details; +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureFromSsl) { + configCertVerificationDetails(false); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyCertChain(std::string(cert_view->subject_alt_name_domains()[0]), 54321, + {leaf_cert_}, ocsp_response, cert_sct, nullptr, + &error_details, nullptr, nullptr)) + << error_details; + EXPECT_EQ("X509_verify_cert: certificate verification error at depth 1: certificate has expired", + error_details); +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidLeafCert) { + configCertVerificationDetails(true); + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + const std::vector certs{"invalid leaf cert"}; + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyCertChain("www.google.com", 54321, certs, ocsp_response, cert_sct, + nullptr, &error_details, nullptr, nullptr)); + EXPECT_EQ("d2i_X509: fail to parse DER", error_details); +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureLeafCertWithGarbage) { + configCertVerificationDetails(true); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + const std::string ocsp_response; + const std::string cert_sct; + std::string cert_with_trailing_garbage = absl::StrCat(leaf_cert_, "AAAAAA"); + std::string error_details; + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyCertChain(std::string(cert_view->subject_alt_name_domains()[0]), 54321, + {cert_with_trailing_garbage}, ocsp_response, cert_sct, + nullptr, &error_details, nullptr, nullptr)) + << error_details; + EXPECT_EQ("There is trailing garbage in DER.", error_details); +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyCertChainFailureInvalidHost) { + configCertVerificationDetails(true); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyCertChain("unknown.org", 54321, {leaf_cert_}, ocsp_response, cert_sct, + nullptr, &error_details, nullptr, nullptr)) + << error_details; + EXPECT_EQ("Leaf certificate doesn't match hostname: unknown.org", error_details); +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyProofFailureEmptyCertChain) { + configCertVerificationDetails(true); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + quic::QuicTransportVersion version{quic::QUIC_VERSION_UNSUPPORTED}; + quiche::QuicheStringPiece chlo_hash{"aaaaa"}; + std::string server_config{"Server Config"}; + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + const std::vector certs; + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyProof(std::string(cert_view->subject_alt_name_domains()[0]), 54321, + server_config, version, chlo_hash, certs, cert_sct, "signature", + nullptr, &error_details, nullptr, nullptr)); + EXPECT_EQ("Received empty cert chain.", error_details); +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyProofFailureInvalidLeafCert) { + configCertVerificationDetails(true); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + quic::QuicTransportVersion version{quic::QUIC_VERSION_UNSUPPORTED}; + quiche::QuicheStringPiece chlo_hash{"aaaaa"}; + std::string server_config{"Server Config"}; + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + const std::vector certs{"invalid leaf cert"}; + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyProof(std::string(cert_view->subject_alt_name_domains()[0]), 54321, + server_config, version, chlo_hash, certs, cert_sct, "signature", + nullptr, &error_details, nullptr, nullptr)); + EXPECT_EQ("Invalid leaf cert.", error_details); +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyProofFailureUnsupportedECKey) { + configCertVerificationDetails(true); + quic::QuicTransportVersion version{quic::QUIC_VERSION_UNSUPPORTED}; + quiche::QuicheStringPiece chlo_hash{"aaaaa"}; + std::string server_config{"Server Config"}; + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + // This is a EC cert with secp384r1 curve which is not supported by Envoy. + const std::string certs{R"(-----BEGIN CERTIFICATE----- +MIICkDCCAhagAwIBAgIUTZbykU9eQL3GdrNlodxrOJDecIQwCgYIKoZIzj0EAwIw +fzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk1BMRIwEAYDVQQHDAlDYW1icmlkZ2Ux +DzANBgNVBAoMBkdvb2dsZTEOMAwGA1UECwwFZW52b3kxDTALBgNVBAMMBHRlc3Qx +HzAdBgkqhkiG9w0BCQEWEGRhbnpoQGdvb2dsZS5jb20wHhcNMjAwODA1MjAyMDI0 +WhcNMjIwODA1MjAyMDI0WjB/MQswCQYDVQQGEwJVUzELMAkGA1UECAwCTUExEjAQ +BgNVBAcMCUNhbWJyaWRnZTEPMA0GA1UECgwGR29vZ2xlMQ4wDAYDVQQLDAVlbnZv +eTENMAsGA1UEAwwEdGVzdDEfMB0GCSqGSIb3DQEJARYQZGFuemhAZ29vZ2xlLmNv +bTB2MBAGByqGSM49AgEGBSuBBAAiA2IABGRaEAtVq+xHXfsF4R/j+mqVN2E29ZYL +oFlvnelKeeT2B51bSfUv+X+Ci1BSa2OxPCVS6o0vpcF6YOlz4CS7QcXZIoRfhsv7 +O2Hz/IdxAPhX/gdK/70T1x+V/6nvIHiiw6NTMFEwHQYDVR0OBBYEFF75rDce6xNJ +GfpKbUg4emG2KWRMMB8GA1UdIwQYMBaAFF75rDce6xNJGfpKbUg4emG2KWRMMA8G +A1UdEwEB/wQFMAMBAf8wCgYIKoZIzj0EAwIDaAAwZQIxAIyZghTK3cmyrRWkxfQ7 +xEc11gujcT8nbytYbM6jodKwcbtR6SOmLx2ychXrCMm2ZAIwXqmrTYBtrbqb3mBx +VdGXMAjeXhnOnPvmDi5hUz/uvI+Pg6cNmUoCRwSCnK/DazhA +-----END CERTIFICATE-----)"}; + std::stringstream pem_stream(certs); + std::vector chain = quic::CertificateView::LoadPemFromStream(&pem_stream); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(chain[0]); + ASSERT(cert_view); + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyProof("www.google.com", 54321, server_config, version, chlo_hash, + chain, cert_sct, "signature", nullptr, &error_details, nullptr, + nullptr)); + EXPECT_EQ("Invalid leaf cert, only P-256 ECDSA certificates are supported", error_details); +} + +TEST_F(EnvoyQuicProofVerifierTest, VerifyProofFailureInvalidSignature) { + configCertVerificationDetails(true); + std::unique_ptr cert_view = + quic::CertificateView::ParseSingleCertificate(leaf_cert_); + quic::QuicTransportVersion version{quic::QUIC_VERSION_UNSUPPORTED}; + quiche::QuicheStringPiece chlo_hash{"aaaaa"}; + std::string server_config{"Server Config"}; + const std::string ocsp_response; + const std::string cert_sct; + std::string error_details; + EXPECT_EQ(quic::QUIC_FAILURE, + verifier_->VerifyProof(std::string(cert_view->subject_alt_name_domains()[0]), 54321, + server_config, version, chlo_hash, {leaf_cert_}, cert_sct, + "signature", nullptr, &error_details, nullptr, nullptr)); + EXPECT_EQ("Signature is not valid.", error_details); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc b/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc index 05fb1e61a7aa..85688dbd0835 100644 --- a/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc +++ b/test/extensions/quic_listeners/quiche/integration/quic_http_integration_test.cc @@ -1,3 +1,5 @@ +#include + #include #include "envoy/config/bootstrap/v3/bootstrap.pb.h" @@ -7,6 +9,7 @@ #include "test/config/utility.h" #include "test/integration/http_integration.h" +#include "test/integration/ssl_utility.h" #include "test/test_common/utility.h" #pragma GCC diagnostic push @@ -23,12 +26,14 @@ #include "extensions/quic_listeners/quiche/envoy_quic_client_session.h" #include "extensions/quic_listeners/quiche/envoy_quic_client_connection.h" -#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h" +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier.h" #include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" #include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" #include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/quic_transport_socket_factory.h" #include "test/extensions/quic_listeners/quiche/test_utils.h" +#include "extensions/transport_sockets/tls/context_config_impl.h" namespace Envoy { namespace Quic { @@ -44,6 +49,43 @@ class CodecClientCallbacksForTest : public Http::CodecClientCallbacks { Http::StreamResetReason last_stream_reset_reason_{Http::StreamResetReason::LocalReset}; }; +std::unique_ptr +createQuicClientTransportSocketFactory(const Ssl::ClientSslTransportOptions& options, Api::Api& api, + const std::string& san_to_match) { + std::string yaml_plain = R"EOF( + common_tls_context: + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/config/integration/certs/cacert.pem" +)EOF"; + envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext tls_context; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), tls_context); + auto* common_context = tls_context.mutable_common_tls_context(); + + if (options.alpn_) { + common_context->add_alpn_protocols("h3"); + } + if (options.san_) { + common_context->mutable_validation_context()->add_match_subject_alt_names()->set_exact( + san_to_match); + } + for (const std::string& cipher_suite : options.cipher_suites_) { + common_context->mutable_tls_params()->add_cipher_suites(cipher_suite); + } + if (!options.sni_.empty()) { + tls_context.set_sni(options.sni_); + } + + common_context->mutable_tls_params()->set_tls_minimum_protocol_version(options.tls_version_); + common_context->mutable_tls_params()->set_tls_maximum_protocol_version(options.tls_version_); + + NiceMock mock_factory_ctx; + ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(api)); + auto cfg = std::make_unique( + tls_context, options.sigalgs_, mock_factory_ctx); + return std::make_unique(std::move(cfg)); +} + class QuicHttpIntegrationTest : public HttpIntegrationTest, public QuicMultiVersionTest { public: QuicHttpIntegrationTest() @@ -59,8 +101,7 @@ class QuicHttpIntegrationTest : public HttpIntegrationTest, public QuicMultiVers SetQuicReloadableFlag(quic_disable_version_draft_25, !use_http3); return quic::CurrentSupportedVersions(); }()), - crypto_config_(std::make_unique()), conn_helper_(*dispatcher_), - alarm_factory_(*dispatcher_, *conn_helper_.GetClock()), + conn_helper_(*dispatcher_), alarm_factory_(*dispatcher_, *conn_helper_.GetClock()), injected_resource_filename_(TestEnvironment::temporaryPath("injected_resource")), file_updater_(injected_resource_filename_) {} @@ -81,7 +122,7 @@ class QuicHttpIntegrationTest : public HttpIntegrationTest, public QuicMultiVers quic::ParsedQuicVersionVector{supported_versions_[0]}, local_addr, *dispatcher_, nullptr); quic_connection_ = connection.get(); auto session = std::make_unique( - quic_config_, supported_versions_, std::move(connection), server_id_, &crypto_config_, + quic_config_, supported_versions_, std::move(connection), server_id_, crypto_config_.get(), &push_promise_index_, *dispatcher_, 0); session->Initialize(); return session; @@ -170,16 +211,24 @@ class QuicHttpIntegrationTest : public HttpIntegrationTest, public QuicMultiVers updateResource(0); HttpIntegrationTest::initialize(); registerTestServerPorts({"http"}); + crypto_config_ = + std::make_unique(std::make_unique( + stats_store_, + createQuicClientTransportSocketFactory( + Ssl::ClientSslTransportOptions().setAlpn(true).setSan(true), *api_, san_to_match_) + ->clientContextConfig(), + timeSystem())); } void updateResource(double pressure) { file_updater_.update(absl::StrCat(pressure)); } protected: quic::QuicConfig quic_config_; - quic::QuicServerId server_id_{"example.com", 443, false}; + quic::QuicServerId server_id_{"lyft.com", 443, false}; + std::string san_to_match_{"spiffe://lyft.com/backend-team"}; quic::QuicClientPushPromiseIndex push_promise_index_; quic::ParsedQuicVersionVector supported_versions_; - quic::QuicCryptoClientConfig crypto_config_; + std::unique_ptr crypto_config_; EnvoyQuicConnectionHelper conn_helper_; EnvoyQuicAlarmFactory alarm_factory_; CodecClientCallbacksForTest client_codec_callback_; @@ -461,5 +510,19 @@ TEST_P(QuicHttpIntegrationTest, AdminDrainDrainsListeners) { testAdminDrain(Http::CodecClient::Type::HTTP1); } +TEST_P(QuicHttpIntegrationTest, CertVerificationFailure) { + san_to_match_ = "www.random_domain.com"; + initialize(); + codec_client_ = makeRawHttpConnection(makeClientConnection((lookupPort("http"))), absl::nullopt); + EXPECT_FALSE(codec_client_->connected()); + std::string failure_reason = + GetParam().second == QuicVersionType::GquicQuicCrypto + ? "QUIC_PROOF_INVALID with details: Proof invalid: X509_verify_cert: certificate " + "verification error at depth 0: ok" + : "QUIC_HANDSHAKE_FAILED with details: TLS handshake failure (ENCRYPTION_HANDSHAKE) 46: " + "certificate unknown"; + EXPECT_EQ(failure_reason, codec_client_->connection()->transportFailureReason()); +} + } // namespace Quic } // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/test_proof_source.h b/test/extensions/quic_listeners/quiche/test_proof_source.h index ad8bae60a540..8b1baf920d69 100644 --- a/test/extensions/quic_listeners/quiche/test_proof_source.h +++ b/test/extensions/quic_listeners/quiche/test_proof_source.h @@ -15,14 +15,14 @@ #include #include "test/mocks/network/mocks.h" -#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h" +#include "extensions/quic_listeners/quiche/envoy_quic_proof_source_base.h" namespace Envoy { namespace Quic { // A test ProofSource which always provide a hard-coded test certificate in // QUICHE and a fake signature. -class TestProofSource : public Quic::EnvoyQuicFakeProofSource { +class TestProofSource : public EnvoyQuicProofSourceBase { public: quic::QuicReferenceCountedPointer GetCertChain(const quic::QuicSocketAddress& /*server_address*/, @@ -31,18 +31,18 @@ class TestProofSource : public Quic::EnvoyQuicFakeProofSource { return cert_chain_; } - void - ComputeTlsSignature(const quic::QuicSocketAddress& /*server_address*/, - const quic::QuicSocketAddress& /*client_address*/, - const std::string& /*hostname*/, uint16_t /*signature_algorithm*/, - quiche::QuicheStringPiece in, - std::unique_ptr callback) override { + const Network::MockFilterChain& filterChain() const { return filter_chain_; } + +protected: + void signPayload(const quic::QuicSocketAddress& /*server_address*/, + const quic::QuicSocketAddress& /*client_address*/, + const std::string& /*hostname*/, uint16_t /*signature_algorithm*/, + quiche::QuicheStringPiece in, + std::unique_ptr callback) override { callback->Run(true, absl::StrCat("Fake signature for { ", in, " }"), std::make_unique(filter_chain_)); } - const Network::MockFilterChain& filterChain() const { return filter_chain_; } - private: quic::QuicReferenceCountedPointer cert_chain_{ new quic::ProofSource::Chain( diff --git a/test/extensions/quic_listeners/quiche/test_proof_verifier.h b/test/extensions/quic_listeners/quiche/test_proof_verifier.h new file mode 100644 index 000000000000..77dada22d1cd --- /dev/null +++ b/test/extensions/quic_listeners/quiche/test_proof_verifier.h @@ -0,0 +1,30 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_proof_verifier_base.h" + +namespace Envoy { +namespace Quic { + +// A test quic::ProofVerifier which always approves the certs and signature. +class TestProofVerifier : public EnvoyQuicProofVerifierBase { +public: + // quic::ProofVerifier + quic::QuicAsyncStatus + VerifyCertChain(const std::string& /*hostname*/, const uint16_t /*port*/, + const std::vector& /*certs*/, const std::string& /*ocsp_response*/, + const std::string& /*cert_sct*/, const quic::ProofVerifyContext* /*context*/, + std::string* /*error_details*/, + std::unique_ptr* /*details*/, + std::unique_ptr /*callback*/) override { + return quic::QUIC_SUCCESS; + } + +protected: + // EnvoyQuicProofVerifierBase + bool verifySignature(const std::string& /*server_config*/, absl::string_view /*chlo_hash*/, + const std::string& /*cert*/, const std::string& /*signature*/, + std::string* /*error_details*/) override { + return true; + } +}; + +} // namespace Quic +} // namespace Envoy diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index c3bc9b2f8ecd..7567e5807cff 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -129,6 +129,23 @@ class MockTlsCertificateConfig : public TlsCertificateConfig { MOCK_METHOD(Envoy::Ssl::PrivateKeyMethodProviderSharedPtr, privateKeyMethod, (), (const)); }; +class MockCertificateValidationContextConfig : public CertificateValidationContextConfig { +public: + MOCK_METHOD(const std::string&, caCert, (), (const)); + MOCK_METHOD(const std::string&, caCertPath, (), (const)); + MOCK_METHOD(const std::string&, certificateRevocationList, (), (const)); + MOCK_METHOD(const std::string&, certificateRevocationListPath, (), (const)); + MOCK_METHOD(const std::vector&, verifySubjectAltNameList, (), (const)); + MOCK_METHOD(const std::vector&, subjectAltNameMatchers, + (), (const)); + MOCK_METHOD(const std::vector&, verifyCertificateHashList, (), (const)); + MOCK_METHOD(const std::vector&, verifyCertificateSpkiList, (), (const)); + MOCK_METHOD(bool, allowExpiredCertificate, (), (const)); + MOCK_METHOD(envoy::extensions::transport_sockets::tls::v3::CertificateValidationContext:: + TrustChainVerification, + trustChainVerification, (), (const)); +}; + class MockPrivateKeyMethodManager : public PrivateKeyMethodManager { public: MockPrivateKeyMethodManager(); diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index d855084a1fd7..cf99f6b3f17a 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -990,6 +990,7 @@ sched schedulable schemas scopekey +secp sendmsg sendmmsg sendto From 739f48476ac29badd2ef6a52cf70a8cfeeeef485 Mon Sep 17 00:00:00 2001 From: ASOP Date: Fri, 7 Aug 2020 14:04:48 -0700 Subject: [PATCH 05/67] Init manager checking unready targets with target-aware watchers (#12035) The current init manager only knows how many registered targets are not ready via its ```count_``` field. This pull request mainly helps init manager to check which specific targets are not ready. Key idea: Init manager stores the **unready_target_name:count** key-value pair in a hash map to check which registered targets are not ready. Enable passing target name into watcher's callback function. 1. When a new target is added into the targets list, increase the occurrence of the target's name in the hash map ```++target_name_count_[target_name]```. 2. When a target is ready, it informs the manager's watcher and finally, the watcher notifies the manager(that's why we need to pass a ```string_view name``` parameter to the manager's callback). And decrease the count of the target's name by 1. ```--target_name_count_[target_name]```. **Most recent update**: Wrapped the old ReadyFn type function into the new ```TargetAwareReadyFn```. Now ```WatcherImpl``` can take both types of function in its Ctor. ~Split WatcherImpl into two classes, the old one and a new ```TargetAwareWatcherImpl```. As we now limit the ```watcher_``` inside ```ManagerImpl``` to receive a ```string_view``` parameter, we declare it to be of ```TargetAwareWatcherImpl```.~ In the current code base, watchers are basically constructed with ```std::function```. In order to adopt the new feature(pass a ```string_view``` parameter to the manager's callback, I wrote a new constructor for ```WatcherImpl``` with ```std::function``` parameter. In addition, I added a ```onTargetReadySendTargetName(string_view)``` adopted from the original callback ```onTargetReady()```. So now both types of callbacks are supported. I've also updated code in the constructor of ```ManagerImpl```, and now the ```watcher_``` of a manager is constructed with the new type of function(with string_view parameter). Therefore, the manager will always get target_name from the watcher's callback. Risk Level: Low Linked Issue: https://github.com/envoyproxy/envoy/issues/11963 Signed-off-by: pingsun --- include/envoy/init/target.h | 6 +++++ source/common/init/manager_impl.cc | 23 ++++++++++++++---- source/common/init/manager_impl.h | 28 ++++++++++++++++------ source/common/init/target_impl.cc | 2 ++ source/common/init/target_impl.h | 2 ++ source/common/init/watcher_impl.cc | 16 ++++++++----- source/common/init/watcher_impl.h | 26 ++++++++++---------- test/integration/stats_integration_test.cc | 8 +++---- 8 files changed, 78 insertions(+), 33 deletions(-) diff --git a/include/envoy/init/target.h b/include/envoy/init/target.h index 9ab46d38aff4..75397ad2f991 100644 --- a/include/envoy/init/target.h +++ b/include/envoy/init/target.h @@ -25,6 +25,12 @@ struct TargetHandle { * @return true if the target received this call, false if the target was already destroyed. */ virtual bool initialize(const Watcher& watcher) const PURE; + + /** + * @return a human-readable target name, for logging / debugging / tracking target names. + * The target name has to be unique. + */ + virtual absl::string_view name() const PURE; }; using TargetHandlePtr = std::unique_ptr; diff --git a/source/common/init/manager_impl.cc b/source/common/init/manager_impl.cc index f60ddc64a9e9..95cb37e4cc3b 100644 --- a/source/common/init/manager_impl.cc +++ b/source/common/init/manager_impl.cc @@ -1,19 +1,23 @@ #include "common/init/manager_impl.h" +#include + #include "common/common/assert.h" +#include "common/init/watcher_impl.h" namespace Envoy { namespace Init { ManagerImpl::ManagerImpl(absl::string_view name) : name_(fmt::format("init manager {}", name)), state_(State::Uninitialized), count_(0), - watcher_(name_, [this]() { onTargetReady(); }) {} + watcher_(name_, [this](absl::string_view target_name) { onTargetReady(target_name); }) {} Manager::State ManagerImpl::state() const { return state_; } void ManagerImpl::add(const Target& target) { ++count_; TargetHandlePtr target_handle(target.createHandle(name_)); + ++target_names_count_[target.name()]; switch (state_) { case State::Uninitialized: // If the manager isn't initialized yet, save the target handle to be initialized later. @@ -53,15 +57,26 @@ void ManagerImpl::initialize(const Watcher& watcher) { // completed immediately. for (const auto& target_handle : target_handles_) { if (!target_handle->initialize(watcher_)) { - onTargetReady(); + onTargetReady(target_handle->name()); } } } } -void ManagerImpl::onTargetReady() { +const absl::flat_hash_map& ManagerImpl::unreadyTargets() const { + return target_names_count_; +} + +void ManagerImpl::onTargetReady(absl::string_view target_name) { // If there are no remaining targets and one mysteriously calls us back, this manager is haunted. - ASSERT(count_ != 0, fmt::format("{} called back by target after initialization complete")); + ASSERT(count_ != 0, + fmt::format("{} called back by target after initialization complete", target_name)); + + // Decrease target_name count by 1. + ASSERT(target_names_count_.find(target_name) != target_names_count_.end()); + if (--target_names_count_[target_name] == 0) { + target_names_count_.erase(target_name); + } // If there are no uninitialized targets remaining when called back by a target, that means it was // the last. Signal `ready` to the handle we saved in `initialize`. diff --git a/source/common/init/manager_impl.h b/source/common/init/manager_impl.h index b92ac102fd72..026014245ccd 100644 --- a/source/common/init/manager_impl.h +++ b/source/common/init/manager_impl.h @@ -7,6 +7,8 @@ #include "common/common/logger.h" #include "common/init/watcher_impl.h" +#include "absl/container/flat_hash_map.h" + namespace Envoy { namespace Init { @@ -35,27 +37,39 @@ class ManagerImpl : public Manager, Logger::Loggable { void add(const Target& target) override; void initialize(const Watcher& watcher) override; + // Expose the const reference of target_names_count_ hash map to public. + const absl::flat_hash_map& unreadyTargets() const; + private: - void onTargetReady(); + // Callback function with an additional target_name parameter, decrease unready targets count by + // 1, update target_names_count_ hash map. + void onTargetReady(absl::string_view target_name); + void ready(); - // Human-readable name for logging + // Human-readable name for logging. const std::string name_; - // Current state + // Current state. State state_; - // Current number of registered targets that have not yet initialized + // Current number of registered targets that have not yet initialized. uint32_t count_; - // Handle to the watcher passed in `initialize`, to be called when initialization completes + // Handle to the watcher passed in `initialize`, to be called when initialization completes. WatcherHandlePtr watcher_handle_; - // Watcher to receive ready notifications from each target + // Watcher to receive ready notifications from each target. We restrict the watcher_ inside + // ManagerImpl to be constructed with the 'TargetAwareReadyFn' fn so that the init manager will + // get target name information when the watcher_ calls 'onTargetSendName(target_name)' For any + // other purpose, a watcher can be constructed with either TargetAwareReadyFn or ReadyFn. const WatcherImpl watcher_; - // All registered targets + // All registered targets. std::list target_handles_; + + // Count of target_name of unready targets. + absl::flat_hash_map target_names_count_; }; } // namespace Init diff --git a/source/common/init/target_impl.cc b/source/common/init/target_impl.cc index 4d8df4c27aac..8ee37eabfd14 100644 --- a/source/common/init/target_impl.cc +++ b/source/common/init/target_impl.cc @@ -22,6 +22,8 @@ bool TargetHandleImpl::initialize(const Watcher& watcher) const { } } +absl::string_view TargetHandleImpl::name() const { return name_; } + TargetImpl::TargetImpl(absl::string_view name, InitializeFn fn) : name_(fmt::format("target {}", name)), fn_(std::make_shared([this, fn](WatcherHandlePtr watcher_handle) { diff --git a/source/common/init/target_impl.h b/source/common/init/target_impl.h index d6a098daaca2..da7281c69b2f 100644 --- a/source/common/init/target_impl.h +++ b/source/common/init/target_impl.h @@ -38,6 +38,8 @@ class TargetHandleImpl : public TargetHandle, Logger::Loggable // Init::TargetHandle bool initialize(const Watcher& watcher) const override; + absl::string_view name() const override; + private: // Name of the handle (almost always the name of the ManagerImpl calling the target) const std::string handle_name_; diff --git a/source/common/init/watcher_impl.cc b/source/common/init/watcher_impl.cc index b69fe3e7cf84..50b792bdcbbe 100644 --- a/source/common/init/watcher_impl.cc +++ b/source/common/init/watcher_impl.cc @@ -4,7 +4,7 @@ namespace Envoy { namespace Init { WatcherHandleImpl::WatcherHandleImpl(absl::string_view handle_name, absl::string_view name, - std::weak_ptr fn) + std::weak_ptr fn) : handle_name_(handle_name), name_(name), fn_(std::move(fn)) {} bool WatcherHandleImpl::ready() const { @@ -12,26 +12,30 @@ bool WatcherHandleImpl::ready() const { if (locked_fn) { // If we can "lock" a shared pointer to the watcher's callback function, call it. ENVOY_LOG(debug, "{} initialized, notifying {}", handle_name_, name_); - (*locked_fn)(); + (*locked_fn)(handle_name_); return true; } else { // If not, the watcher was already destroyed. - ENVOY_LOG(debug, "{} initialized, but can't notify {} (unavailable)", handle_name_, name_); + ENVOY_LOG(debug, "{} initialized, but can't notify {}", handle_name_, name_); return false; } } WatcherImpl::WatcherImpl(absl::string_view name, ReadyFn fn) - : name_(name), fn_(std::make_shared(std::move(fn))) {} + : name_(name), fn_(std::make_shared( + [callback = std::move(fn)](absl::string_view) { callback(); })) {} + +WatcherImpl::WatcherImpl(absl::string_view name, TargetAwareReadyFn fn) + : name_(name), fn_(std::make_shared(std::move(fn))) {} WatcherImpl::~WatcherImpl() { ENVOY_LOG(debug, "{} destroyed", name_); } absl::string_view WatcherImpl::name() const { return name_; } WatcherHandlePtr WatcherImpl::createHandle(absl::string_view handle_name) const { - // Note: can't use std::make_unique because WatcherHandleImpl ctor is private + // Note: can't use std::make_unique because WatcherHandleImpl ctor is private. return std::unique_ptr( - new WatcherHandleImpl(handle_name, name_, std::weak_ptr(fn_))); + new WatcherHandleImpl(handle_name, name_, std::weak_ptr(fn_))); } } // namespace Init diff --git a/source/common/init/watcher_impl.h b/source/common/init/watcher_impl.h index 816a37c860eb..fb41d8c0400a 100644 --- a/source/common/init/watcher_impl.h +++ b/source/common/init/watcher_impl.h @@ -14,6 +14,7 @@ namespace Init { * initialization completes. */ using ReadyFn = std::function; +using TargetAwareReadyFn = std::function; /** * A WatcherHandleImpl functions as a weak reference to a Watcher. It is how a TargetImpl safely @@ -25,22 +26,22 @@ class WatcherHandleImpl : public WatcherHandle, Logger::Loggable fn); + std::weak_ptr fn); public: - // Init::WatcherHandle + // Init::WatcherHandle. bool ready() const override; private: // Name of the handle (either the name of the target calling the manager, or the name of the - // manager calling the client) + // manager calling the client). const std::string handle_name_; - // Name of the watcher (either the name of the manager, or the name of the client) + // Name of the watcher (either the name of the manager, or the name of the client). const std::string name_; - // The watcher's callback function, only called if the weak pointer can be "locked" - const std::weak_ptr fn_; + // The watcher's callback function, only called if the weak pointer can be "locked". + const std::weak_ptr fn_; }; /** @@ -51,22 +52,23 @@ class WatcherHandleImpl : public WatcherHandle, Logger::Loggable { public: /** - * @param name a human-readable watcher name, for logging / debugging - * @param fn a callback function to invoke when `ready` is called on the handle + * @param name a human-readable watcher name, for logging / debugging. + * @param fn a callback function to invoke when `ready` is called on the handle. */ WatcherImpl(absl::string_view name, ReadyFn fn); + WatcherImpl(absl::string_view name, TargetAwareReadyFn fn); ~WatcherImpl() override; - // Init::Watcher + // Init::Watcher. absl::string_view name() const override; WatcherHandlePtr createHandle(absl::string_view handle_name) const override; private: - // Human-readable name for logging + // Human-readable name for logging. const std::string name_; - // The callback function, called via WatcherHandleImpl by either the target or the manager - const std::shared_ptr fn_; + // The callback function, called via WatcherHandleImpl by either the target or the manager. + const std::shared_ptr fn_; }; } // namespace Init diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 1238176409b3..26143f370000 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -286,6 +286,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // 2020/07/20 11559 44747 46000 stats: add histograms for request/response headers // and body sizes. // 2020/07/21 12034 44811 46000 Add configurable histogram buckets. + // 2020/07/31 12035 45002 46000 Init manager store unready targets in hash map. // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -303,8 +304,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // We only run the exact test for ipv6 because ipv4 in some cases may allocate a // different number of bytes. We still run the approximate test. if (ip_version_ != Network::Address::IpVersion::v6) { - // https://github.com/envoyproxy/envoy/issues/12209 - // EXPECT_MEMORY_EQ(m_per_cluster, 44811); + EXPECT_MEMORY_EQ(m_per_cluster, 45002); } EXPECT_MEMORY_LE(m_per_cluster, 46000); // Round up to allow platform variations. } @@ -362,6 +362,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // 2020/07/20 11559 36859 38000 stats: add histograms for request/response headers // and body sizes. // 2020/07/21 12034 36923 38000 Add configurable histogram buckets. + // 2020/07/31 12035 37114 38000 Init manager store unready targets in hash map. // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -379,8 +380,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // We only run the exact test for ipv6 because ipv4 in some cases may allocate a // different number of bytes. We still run the approximate test. if (ip_version_ != Network::Address::IpVersion::v6) { - // https://github.com/envoyproxy/envoy/issues/12209 - // EXPECT_MEMORY_EQ(m_per_cluster, 36923); + EXPECT_MEMORY_EQ(m_per_cluster, 37114); } EXPECT_MEMORY_LE(m_per_cluster, 38000); // Round up to allow platform variations. } From bd40556d20b14bb947956a0093719b0d986bcea7 Mon Sep 17 00:00:00 2001 From: DongRyeol Cha Date: Sat, 8 Aug 2020 07:52:45 +0900 Subject: [PATCH 06/67] Fix potential bug that call the virtual method in destructor (#12536) As you know that the virtual method should not be called in destructor because it is not guaranteed if virtual function table exist on destructor stage. There are no crash or abnormal behavior now but it is definitely a potential bug. So, this patch changes the virtual function calling to non virtual function calling. Signed-off-by: DongRyeol Cha --- source/common/network/udp_listener_impl.cc | 6 ++++-- source/common/network/udp_listener_impl.h | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index c959132d1642..2b28e0ea5841 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -45,16 +45,18 @@ UdpListenerImpl::UdpListenerImpl(Event::DispatcherImpl& dispatcher, SocketShared } UdpListenerImpl::~UdpListenerImpl() { - disable(); + disableEvent(); file_event_.reset(); } -void UdpListenerImpl::disable() { file_event_->setEnabled(0); } +void UdpListenerImpl::disable() { disableEvent(); } void UdpListenerImpl::enable() { file_event_->setEnabled(Event::FileReadyType::Read | Event::FileReadyType::Write); } +void UdpListenerImpl::disableEvent() { file_event_->setEnabled(0); } + void UdpListenerImpl::onSocketEvent(short flags) { ASSERT((flags & (Event::FileReadyType::Read | Event::FileReadyType::Write))); ENVOY_UDP_LOG(trace, "socket event: {}", flags); diff --git a/source/common/network/udp_listener_impl.h b/source/common/network/udp_listener_impl.h index 2184b4419c10..9789301361c3 100644 --- a/source/common/network/udp_listener_impl.h +++ b/source/common/network/udp_listener_impl.h @@ -54,6 +54,7 @@ class UdpListenerImpl : public BaseListenerImpl, private: void onSocketEvent(short flags); + void disableEvent(); TimeSource& time_source_; Event::FileEventPtr file_event_; From 933e267db60961e76549ae6e65ac4872cb78db4d Mon Sep 17 00:00:00 2001 From: yugantrana Date: Fri, 7 Aug 2020 21:06:33 -0400 Subject: [PATCH 07/67] udp: write performance improvement via udp_gso (#12219) Introduces UdpPacketWriter Interface that can be used to perform writes in Batched/PassThrough modes by using QuicGsoBatchWriter implementation from QUICHE extension. **Additional Description:** UDP GSO (Generic Segmentation Offload) was introduced in Linux at version 4.18. It allows batch-writing of multiple messages into a single payload and sending these messages along as a batch in a single sendmsg syscall. Currently, Envoy performs the sending of messages using simple sendmsg implementation in pass-through mode, i.e. no support for batch writing. With this change, UdpListener can use UdpPacketWriter interface as a DefaultWriter or a GsoBatchWriter to perform pass-through or batched writes respectively. Detailed description of the changes can be found in the design document, [here](https://docs.google.com/document/d/16ePbgkfrzQ6v-cOVMSnKDja3dUdZvX-mxT9jw29rx4g/edit?usp=sharing). **Risk Level:** Low, not in use **Testing:** - Added udp_listener_impl_batched_writes_test, to verify that multiple packets of varying sizes are batched/flushed as per gso specifications while using UdpGsoBatchWriter. - Modified existing tests, to verify that UdpDefaultWriter performs writes in pass-through mode. - Ran all tests. All 677 tests passed successfully. ``` **Docs Changes:** None **Release Notes:** None **Fixes:** #11925 Signed-off-by: Yugant --- api/envoy/config/listener/v3/listener.proto | 13 +- .../v3/udp_default_writer_config.proto | 21 ++ .../v3/udp_gso_batch_writer_config.proto | 21 ++ .../config/listener/v4alpha/listener.proto | 13 +- .../v4alpha/udp_default_writer_config.proto | 23 ++ .../v4alpha/udp_gso_batch_writer_config.proto | 23 ++ bazel/external/quiche.BUILD | 14 + .../envoy/config/listener/v3/listener.proto | 13 +- .../v3/udp_default_writer_config.proto | 21 ++ .../v3/udp_gso_batch_writer_config.proto | 21 ++ .../config/listener/v4alpha/listener.proto | 13 +- .../v4alpha/udp_default_writer_config.proto | 23 ++ .../v4alpha/udp_gso_batch_writer_config.proto | 23 ++ include/envoy/api/os_sys_calls.h | 5 + include/envoy/common/platform.h | 12 + include/envoy/network/BUILD | 24 ++ include/envoy/network/listener.h | 21 ++ .../envoy/network/udp_packet_writer_config.h | 26 ++ .../envoy/network/udp_packet_writer_handler.h | 120 ++++++++ source/common/api/posix/os_sys_calls_impl.cc | 21 +- source/common/api/posix/os_sys_calls_impl.h | 1 + source/common/api/win32/os_sys_calls_impl.cc | 5 + source/common/api/win32/os_sys_calls_impl.h | 1 + source/common/network/BUILD | 27 ++ .../network/udp_default_writer_config.cc | 32 ++ .../network/udp_default_writer_config.h | 32 ++ source/common/network/udp_listener_impl.cc | 10 +- source/common/network/udp_listener_impl.h | 1 + .../network/udp_packet_writer_handler_impl.cc | 28 ++ .../network/udp_packet_writer_handler_impl.h | 45 +++ .../filters/udp/udp_proxy/udp_proxy_filter.cc | 2 + source/extensions/quic_listeners/quiche/BUILD | 63 +++- .../quiche/active_quic_listener.cc | 22 +- .../quiche/active_quic_listener.h | 2 + .../quiche/envoy_quic_client_connection.cc | 18 +- .../quiche/envoy_quic_packet_writer.cc | 76 +++-- .../quiche/envoy_quic_packet_writer.h | 31 +- .../quic_listeners/quiche/envoy_quic_utils.cc | 21 +- .../quic_listeners/quiche/envoy_quic_utils.h | 3 +- .../quiche/udp_gso_batch_writer.cc | 126 ++++++++ .../quiche/udp_gso_batch_writer.h | 124 ++++++++ .../quiche/udp_gso_batch_writer_config.cc | 30 ++ .../quiche/udp_gso_batch_writer_config.h | 28 ++ source/server/BUILD | 1 + source/server/admin/admin.h | 3 + source/server/connection_handler_impl.cc | 31 +- source/server/connection_handler_impl.h | 11 +- source/server/listener_impl.cc | 21 ++ source/server/listener_impl.h | 5 + test/common/network/BUILD | 51 ++++ .../udp_listener_impl_batch_writer_test.cc | 279 ++++++++++++++++++ test/common/network/udp_listener_impl_test.cc | 116 ++------ .../network/udp_listener_impl_test_base.h | 123 ++++++++ .../proxy_protocol_regression_test.cc | 1 + .../proxy_protocol/proxy_protocol_test.cc | 2 + .../udp/udp_proxy/udp_proxy_filter_test.cc | 5 + test/extensions/quic_listeners/quiche/BUILD | 2 + .../quiche/active_quic_listener_test.cc | 18 +- .../quiche/envoy_quic_dispatcher_test.cc | 6 +- .../quiche/envoy_quic_utils_test.cc | 2 +- .../quiche/envoy_quic_writer_test.cc | 4 +- test/integration/fake_upstream.h | 8 +- test/mocks/network/mocks.h | 27 ++ test/server/BUILD | 2 + test/server/connection_handler_test.cc | 11 +- .../listener_manager_impl_quic_only_test.cc | 26 +- test/server/listener_manager_impl_test.cc | 22 ++ tools/spelling/spelling_dictionary.txt | 4 + 68 files changed, 1792 insertions(+), 188 deletions(-) create mode 100644 api/envoy/config/listener/v3/udp_default_writer_config.proto create mode 100644 api/envoy/config/listener/v3/udp_gso_batch_writer_config.proto create mode 100644 api/envoy/config/listener/v4alpha/udp_default_writer_config.proto create mode 100644 api/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto create mode 100644 generated_api_shadow/envoy/config/listener/v3/udp_default_writer_config.proto create mode 100644 generated_api_shadow/envoy/config/listener/v3/udp_gso_batch_writer_config.proto create mode 100644 generated_api_shadow/envoy/config/listener/v4alpha/udp_default_writer_config.proto create mode 100644 generated_api_shadow/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto create mode 100644 include/envoy/network/udp_packet_writer_config.h create mode 100644 include/envoy/network/udp_packet_writer_handler.h create mode 100644 source/common/network/udp_default_writer_config.cc create mode 100644 source/common/network/udp_default_writer_config.h create mode 100644 source/common/network/udp_packet_writer_handler_impl.cc create mode 100644 source/common/network/udp_packet_writer_handler_impl.h create mode 100644 source/extensions/quic_listeners/quiche/udp_gso_batch_writer.cc create mode 100644 source/extensions/quic_listeners/quiche/udp_gso_batch_writer.h create mode 100644 source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.cc create mode 100644 source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.h create mode 100644 test/common/network/udp_listener_impl_batch_writer_test.cc create mode 100644 test/common/network/udp_listener_impl_test_base.h diff --git a/api/envoy/config/listener/v3/listener.proto b/api/envoy/config/listener/v3/listener.proto index ab0b0ecac7c7..8c5066909caf 100644 --- a/api/envoy/config/listener/v3/listener.proto +++ b/api/envoy/config/listener/v3/listener.proto @@ -5,6 +5,7 @@ package envoy.config.listener.v3; import "envoy/config/accesslog/v3/accesslog.proto"; import "envoy/config/core/v3/address.proto"; import "envoy/config/core/v3/base.proto"; +import "envoy/config/core/v3/extension.proto"; import "envoy/config/core/v3/socket_option.proto"; import "envoy/config/listener/v3/api_listener.proto"; import "envoy/config/listener/v3/listener_components.proto"; @@ -35,7 +36,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 23] +// [#next-free-field: 24] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener"; @@ -248,4 +249,14 @@ message Listener { // Configuration for :ref:`access logs ` // emitted by this listener. repeated accesslog.v3.AccessLog access_log = 22; + + // If the protocol in the listener socket address in :ref:`protocol + // ` is :ref:`UDP + // `, this field specifies the actual udp + // writer to create, i.e. :ref:`name ` + // = "udp_default_writer" for creating a udp writer with writing in passthrough mode, + // = "udp_gso_batch_writer" for creating a udp writer with writing in batch mode. + // If not present, treat it as "udp_default_writer". + // [#not-implemented-hide:] + core.v3.TypedExtensionConfig udp_writer_config = 23; } diff --git a/api/envoy/config/listener/v3/udp_default_writer_config.proto b/api/envoy/config/listener/v3/udp_default_writer_config.proto new file mode 100644 index 000000000000..707a66c7b5c4 --- /dev/null +++ b/api/envoy/config/listener/v3/udp_default_writer_config.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.listener.v3; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v3"; +option java_outer_classname = "UdpDefaultWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Udp Default Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Default Writer. +message UdpDefaultWriterOptions { +} diff --git a/api/envoy/config/listener/v3/udp_gso_batch_writer_config.proto b/api/envoy/config/listener/v3/udp_gso_batch_writer_config.proto new file mode 100644 index 000000000000..134cb6a42dd2 --- /dev/null +++ b/api/envoy/config/listener/v3/udp_gso_batch_writer_config.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.listener.v3; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v3"; +option java_outer_classname = "UdpGsoBatchWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Udp Gso Batch Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Gso Batch Writer. +message UdpGsoBatchWriterOptions { +} diff --git a/api/envoy/config/listener/v4alpha/listener.proto b/api/envoy/config/listener/v4alpha/listener.proto index 7c8c92fc4989..c188ecb24490 100644 --- a/api/envoy/config/listener/v4alpha/listener.proto +++ b/api/envoy/config/listener/v4alpha/listener.proto @@ -5,6 +5,7 @@ package envoy.config.listener.v4alpha; import "envoy/config/accesslog/v4alpha/accesslog.proto"; import "envoy/config/core/v4alpha/address.proto"; import "envoy/config/core/v4alpha/base.proto"; +import "envoy/config/core/v4alpha/extension.proto"; import "envoy/config/core/v4alpha/socket_option.proto"; import "envoy/config/listener/v4alpha/api_listener.proto"; import "envoy/config/listener/v4alpha/listener_components.proto"; @@ -38,7 +39,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 23] +// [#next-free-field: 24] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.config.listener.v3.Listener"; @@ -251,4 +252,14 @@ message Listener { // Configuration for :ref:`access logs ` // emitted by this listener. repeated accesslog.v4alpha.AccessLog access_log = 22; + + // If the protocol in the listener socket address in :ref:`protocol + // ` is :ref:`UDP + // `, this field specifies the actual udp + // writer to create, i.e. :ref:`name ` + // = "udp_default_writer" for creating a udp writer with writing in passthrough mode, + // = "udp_gso_batch_writer" for creating a udp writer with writing in batch mode. + // If not present, treat it as "udp_default_writer". + // [#not-implemented-hide:] + core.v4alpha.TypedExtensionConfig udp_writer_config = 23; } diff --git a/api/envoy/config/listener/v4alpha/udp_default_writer_config.proto b/api/envoy/config/listener/v4alpha/udp_default_writer_config.proto new file mode 100644 index 000000000000..02660a7b49f4 --- /dev/null +++ b/api/envoy/config/listener/v4alpha/udp_default_writer_config.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.config.listener.v4alpha; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v4alpha"; +option java_outer_classname = "UdpDefaultWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Udp Default Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Default Writer. +message UdpDefaultWriterOptions { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.listener.v3.UdpDefaultWriterOptions"; +} diff --git a/api/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto b/api/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto new file mode 100644 index 000000000000..5427fe19e7e1 --- /dev/null +++ b/api/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.config.listener.v4alpha; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v4alpha"; +option java_outer_classname = "UdpGsoBatchWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Udp Gso Batch Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Gso Batch Writer. +message UdpGsoBatchWriterOptions { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.listener.v3.UdpGsoBatchWriterOptions"; +} diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 50f9f8443c21..b641e9d59e84 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -3584,6 +3584,20 @@ envoy_cc_test_library( deps = [":quic_core_crypto_random_lib"], ) +envoy_cc_test_library( + name = "quic_test_tools_mock_syscall_wrapper_lib", + srcs = ["quiche/quic/test_tools/quic_mock_syscall_wrapper.cc"], + hdrs = ["quiche/quic/test_tools/quic_mock_syscall_wrapper.h"], + copts = quiche_copts, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_syscall_wrapper_lib", + ":quic_platform_base", + ":quic_platform_test", + ], +) + envoy_cc_test_library( name = "quic_test_tools_sent_packet_manager_peer_lib", srcs = ["quiche/quic/test_tools/quic_sent_packet_manager_peer.cc"], diff --git a/generated_api_shadow/envoy/config/listener/v3/listener.proto b/generated_api_shadow/envoy/config/listener/v3/listener.proto index fbf34d16442b..0d0dc5d817a9 100644 --- a/generated_api_shadow/envoy/config/listener/v3/listener.proto +++ b/generated_api_shadow/envoy/config/listener/v3/listener.proto @@ -5,6 +5,7 @@ package envoy.config.listener.v3; import "envoy/config/accesslog/v3/accesslog.proto"; import "envoy/config/core/v3/address.proto"; import "envoy/config/core/v3/base.proto"; +import "envoy/config/core/v3/extension.proto"; import "envoy/config/core/v3/socket_option.proto"; import "envoy/config/listener/v3/api_listener.proto"; import "envoy/config/listener/v3/listener_components.proto"; @@ -35,7 +36,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 23] +// [#next-free-field: 24] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener"; @@ -247,5 +248,15 @@ message Listener { // emitted by this listener. repeated accesslog.v3.AccessLog access_log = 22; + // If the protocol in the listener socket address in :ref:`protocol + // ` is :ref:`UDP + // `, this field specifies the actual udp + // writer to create, i.e. :ref:`name ` + // = "udp_default_writer" for creating a udp writer with writing in passthrough mode, + // = "udp_gso_batch_writer" for creating a udp writer with writing in batch mode. + // If not present, treat it as "udp_default_writer". + // [#not-implemented-hide:] + core.v3.TypedExtensionConfig udp_writer_config = 23; + google.protobuf.BoolValue hidden_envoy_deprecated_use_original_dst = 4 [deprecated = true]; } diff --git a/generated_api_shadow/envoy/config/listener/v3/udp_default_writer_config.proto b/generated_api_shadow/envoy/config/listener/v3/udp_default_writer_config.proto new file mode 100644 index 000000000000..707a66c7b5c4 --- /dev/null +++ b/generated_api_shadow/envoy/config/listener/v3/udp_default_writer_config.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.listener.v3; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v3"; +option java_outer_classname = "UdpDefaultWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Udp Default Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Default Writer. +message UdpDefaultWriterOptions { +} diff --git a/generated_api_shadow/envoy/config/listener/v3/udp_gso_batch_writer_config.proto b/generated_api_shadow/envoy/config/listener/v3/udp_gso_batch_writer_config.proto new file mode 100644 index 000000000000..134cb6a42dd2 --- /dev/null +++ b/generated_api_shadow/envoy/config/listener/v3/udp_gso_batch_writer_config.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.listener.v3; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v3"; +option java_outer_classname = "UdpGsoBatchWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Udp Gso Batch Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Gso Batch Writer. +message UdpGsoBatchWriterOptions { +} diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto b/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto index 7c8c92fc4989..c188ecb24490 100644 --- a/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto +++ b/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto @@ -5,6 +5,7 @@ package envoy.config.listener.v4alpha; import "envoy/config/accesslog/v4alpha/accesslog.proto"; import "envoy/config/core/v4alpha/address.proto"; import "envoy/config/core/v4alpha/base.proto"; +import "envoy/config/core/v4alpha/extension.proto"; import "envoy/config/core/v4alpha/socket_option.proto"; import "envoy/config/listener/v4alpha/api_listener.proto"; import "envoy/config/listener/v4alpha/listener_components.proto"; @@ -38,7 +39,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 23] +// [#next-free-field: 24] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.config.listener.v3.Listener"; @@ -251,4 +252,14 @@ message Listener { // Configuration for :ref:`access logs ` // emitted by this listener. repeated accesslog.v4alpha.AccessLog access_log = 22; + + // If the protocol in the listener socket address in :ref:`protocol + // ` is :ref:`UDP + // `, this field specifies the actual udp + // writer to create, i.e. :ref:`name ` + // = "udp_default_writer" for creating a udp writer with writing in passthrough mode, + // = "udp_gso_batch_writer" for creating a udp writer with writing in batch mode. + // If not present, treat it as "udp_default_writer". + // [#not-implemented-hide:] + core.v4alpha.TypedExtensionConfig udp_writer_config = 23; } diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/udp_default_writer_config.proto b/generated_api_shadow/envoy/config/listener/v4alpha/udp_default_writer_config.proto new file mode 100644 index 000000000000..02660a7b49f4 --- /dev/null +++ b/generated_api_shadow/envoy/config/listener/v4alpha/udp_default_writer_config.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.config.listener.v4alpha; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v4alpha"; +option java_outer_classname = "UdpDefaultWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Udp Default Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Default Writer. +message UdpDefaultWriterOptions { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.listener.v3.UdpDefaultWriterOptions"; +} diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto b/generated_api_shadow/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto new file mode 100644 index 000000000000..5427fe19e7e1 --- /dev/null +++ b/generated_api_shadow/envoy/config/listener/v4alpha/udp_gso_batch_writer_config.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package envoy.config.listener.v4alpha; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; + +option java_package = "io.envoyproxy.envoy.config.listener.v4alpha"; +option java_outer_classname = "UdpGsoBatchWriterConfigProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Udp Gso Batch Writer Config] + +// [#not-implemented-hide:] +// Configuration specific to the Udp Gso Batch Writer. +message UdpGsoBatchWriterOptions { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.listener.v3.UdpGsoBatchWriterOptions"; +} diff --git a/include/envoy/api/os_sys_calls.h b/include/envoy/api/os_sys_calls.h index 28dd3d305652..071a5465d5b5 100644 --- a/include/envoy/api/os_sys_calls.h +++ b/include/envoy/api/os_sys_calls.h @@ -67,6 +67,11 @@ class OsSysCalls { */ virtual bool supportsUdpGro() const PURE; + /** + * return true if the OS supports UDP GSO + */ + virtual bool supportsUdpGso() const PURE; + /** * Release all resources allocated for fd. * @return zero on success, -1 returned otherwise. diff --git a/include/envoy/common/platform.h b/include/envoy/common/platform.h index 30da6aa87039..71e0795c9a55 100644 --- a/include/envoy/common/platform.h +++ b/include/envoy/common/platform.h @@ -189,6 +189,18 @@ struct msghdr { #define IP6T_SO_ORIGINAL_DST 80 #endif +#ifndef SOL_UDP +#define SOL_UDP 17 +#endif + +#ifndef UDP_GRO +#define UDP_GRO 104 +#endif + +#ifndef UDP_SEGMENT +#define UDP_SEGMENT 103 +#endif + typedef int os_fd_t; #define INVALID_SOCKET -1 diff --git a/include/envoy/network/BUILD b/include/envoy/network/BUILD index 3076f862ddb8..3a8e67613c58 100644 --- a/include/envoy/network/BUILD +++ b/include/envoy/network/BUILD @@ -41,6 +41,20 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "udp_packet_writer_handler_interface", + hdrs = ["udp_packet_writer_handler.h"], + deps = [ + ":address_interface", + ":io_handle_interface", + ":socket_interface", + "//include/envoy/api:io_error_interface", + "//include/envoy/buffer:buffer_interface", + "//include/envoy/stats:stats_interface", + "//include/envoy/stats:stats_macros", + ], +) + envoy_cc_library( name = "dns_interface", hdrs = ["dns.h"], @@ -137,6 +151,7 @@ envoy_cc_library( ":connection_balancer_interface", ":connection_interface", ":listen_socket_interface", + ":udp_packet_writer_handler_interface", "//include/envoy/access_log:access_log_interface", "//include/envoy/common:resource_interface", "//include/envoy/stats:stats_interface", @@ -154,6 +169,15 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "udp_packet_writer_config_interface", + hdrs = ["udp_packet_writer_config.h"], + deps = [ + "//include/envoy/config:typed_config_interface", + "//include/envoy/network:udp_packet_writer_handler_interface", + ], +) + envoy_cc_library( name = "proxy_protocol_options_lib", hdrs = ["proxy_protocol.h"], diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 373f25caaf2c..3d8257e69c5f 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -12,6 +12,7 @@ #include "envoy/network/connection.h" #include "envoy/network/connection_balancer.h" #include "envoy/network/listen_socket.h" +#include "envoy/network/udp_packet_writer_handler.h" #include "envoy/stats/scope.h" namespace Envoy { @@ -134,6 +135,12 @@ class ListenerConfig { */ virtual ActiveUdpListenerFactory* udpListenerFactory() PURE; + /** + * @return factory pointer if writing on UDP socket, otherwise return + * nullptr. + */ + virtual UdpPacketWriterFactoryOptRef udpPacketWriterFactory() PURE; + /** * @return traffic direction of the listener. */ @@ -254,6 +261,12 @@ class UdpListenerCallbacks { * @param error_code supplies the received error on the listener. */ virtual void onReceiveError(Api::IoError::IoErrorCode error_code) PURE; + + /** + * Returns the pointer to the udp_packet_writer associated with the + * UdpListenerCallback + */ + virtual UdpPacketWriter& udpPacketWriter() PURE; }; /** @@ -305,6 +318,14 @@ class UdpListener : public virtual Listener { * sender. */ virtual Api::IoCallUint64Result send(const UdpSendData& data) PURE; + + /** + * Flushes out remaining buffered data since last call of send(). + * This is a no-op if the implementation doesn't buffer data while sending. + * + * @return the error code of the underlying flush api. + */ + virtual Api::IoCallUint64Result flush() PURE; }; using UdpListenerPtr = std::unique_ptr; diff --git a/include/envoy/network/udp_packet_writer_config.h b/include/envoy/network/udp_packet_writer_config.h new file mode 100644 index 000000000000..dee4487e2198 --- /dev/null +++ b/include/envoy/network/udp_packet_writer_config.h @@ -0,0 +1,26 @@ +#pragma once + +#include "envoy/config/typed_config.h" +#include "envoy/network/udp_packet_writer_handler.h" + +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Network { + +class UdpPacketWriterConfigFactory : public Config::TypedFactory { +public: + ~UdpPacketWriterConfigFactory() override = default; + + /** + * Create an UdpPacketWriterFactory object according to given message. + * @param message specifies Udp Packet Writer options in a protobuf. + */ + virtual Network::UdpPacketWriterFactoryPtr + createUdpPacketWriterFactory(const Protobuf::Message& message) PURE; + + std::string category() const override { return "envoy.udp_packet_writers"; } +}; + +} // namespace Network +} // namespace Envoy diff --git a/include/envoy/network/udp_packet_writer_handler.h b/include/envoy/network/udp_packet_writer_handler.h new file mode 100644 index 000000000000..dc82e54d8c34 --- /dev/null +++ b/include/envoy/network/udp_packet_writer_handler.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include + +#include "envoy/api/io_error.h" +#include "envoy/buffer/buffer.h" +#include "envoy/network/address.h" +#include "envoy/network/socket.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" + +namespace Envoy { +namespace Network { + +/** + * Max v6 packet size, excluding IP and UDP headers. + */ +constexpr uint64_t UdpMaxOutgoingPacketSize = 1452; + +/** + * UdpPacketWriterBuffer bundles a buffer and a function that + * releases it. + */ +struct UdpPacketWriterBuffer { + UdpPacketWriterBuffer() = default; + UdpPacketWriterBuffer(uint8_t* buffer, size_t length, + std::function release_buffer) + : buffer_(buffer), length_(length), release_buffer_(std::move(release_buffer)) {} + + uint8_t* buffer_ = nullptr; + size_t length_ = 0; + std::function release_buffer_; +}; + +class UdpPacketWriter { +public: + virtual ~UdpPacketWriter() = default; + + /** + * @brief Sends a packet via given UDP socket with specific source address. + * + * @param buffer points to the buffer containing the packet + * @param local_ip is the source address to be used to send. If it is null, + * picks up the default network interface ip address. + * @param peer_address is the destination address to send to. + * @return result with number of bytes written, and write status + */ + virtual Api::IoCallUint64Result writePacket(const Buffer::Instance& buffer, + const Address::Ip* local_ip, + const Address::Instance& peer_address) PURE; + + /** + * @returns true if the network socket is not writable. + */ + virtual bool isWriteBlocked() const PURE; + + /** + * @brief mark the socket as writable when the socket is unblocked. + */ + virtual void setWritable() PURE; + + /** + * @brief Get the maximum size of the packet which can be written using this + * writer for the supplied peer address. + * + * @param peer_address is the destination address to send to. + * @return the max packet size + */ + virtual uint64_t getMaxPacketSize(const Address::Instance& peer_address) const PURE; + + /** + * @return true if Batch Mode + * @return false if PassThroughMode + */ + virtual bool isBatchMode() const PURE; + + /** + * @brief Get pointer to the next write location in internal buffer, + * it should be called iff the caller does not call writePacket + * for the returned buffer. The caller is expected to call writePacket + * with the buffer returned from this function to save a memcpy. + * + * @param local_ip is the source address to be used to send. + * @param peer_address is the destination address to send to. + * @return { char* to the next write location, + * func to release buffer } + */ + virtual UdpPacketWriterBuffer getNextWriteLocation(const Address::Ip* local_ip, + const Address::Instance& peer_address) PURE; + + /** + * @brief Batch Mode: Try to send all buffered packets + * PassThrough Mode: NULL operation + * + * @return Api::IoCallUint64Result + */ + virtual Api::IoCallUint64Result flush() PURE; +}; + +using UdpPacketWriterPtr = std::unique_ptr; + +class UdpPacketWriterFactory { +public: + virtual ~UdpPacketWriterFactory() = default; + + /** + * Creates an UdpPacketWriter object for the given Udp Socket + * @param socket UDP socket used to send packets. + * @return the UdpPacketWriter created. + */ + virtual UdpPacketWriterPtr createUdpPacketWriter(Network::IoHandle& io_handle, + Stats::Scope& scope) PURE; +}; + +using UdpPacketWriterFactoryPtr = std::unique_ptr; +using UdpPacketWriterFactoryOptRef = absl::optional>; + +} // namespace Network +} // namespace Envoy diff --git a/source/common/api/posix/os_sys_calls_impl.cc b/source/common/api/posix/os_sys_calls_impl.cc index e1366dbbf942..546015123bc0 100644 --- a/source/common/api/posix/os_sys_calls_impl.cc +++ b/source/common/api/posix/os_sys_calls_impl.cc @@ -76,9 +76,6 @@ bool OsSysCallsImpl::supportsMmsg() const { bool OsSysCallsImpl::supportsUdpGro() const { #if !defined(__linux__) return false; -#else -#ifndef UDP_GRO - return false; #else static const bool is_supported = [] { int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); @@ -92,6 +89,24 @@ bool OsSysCallsImpl::supportsUdpGro() const { }(); return is_supported; #endif +} + +bool OsSysCallsImpl::supportsUdpGso() const { +#if !defined(__linux__) + return false; +#else + static const bool is_supported = [] { + int fd = ::socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, IPPROTO_UDP); + if (fd < 0) { + return false; + } + int optval; + socklen_t optlen = sizeof(optval); + bool result = (0 <= ::getsockopt(fd, IPPROTO_UDP, UDP_SEGMENT, &optval, &optlen)); + ::close(fd); + return result; + }(); + return is_supported; #endif } diff --git a/source/common/api/posix/os_sys_calls_impl.h b/source/common/api/posix/os_sys_calls_impl.h index a35b15113806..036604eb40c1 100644 --- a/source/common/api/posix/os_sys_calls_impl.h +++ b/source/common/api/posix/os_sys_calls_impl.h @@ -23,6 +23,7 @@ class OsSysCallsImpl : public OsSysCalls { struct timespec* timeout) override; bool supportsMmsg() const override; bool supportsUdpGro() const override; + bool supportsUdpGso() const override; SysCallIntResult close(os_fd_t fd) override; SysCallIntResult ftruncate(int fd, off_t length) override; SysCallPtrResult mmap(void* addr, size_t length, int prot, int flags, int fd, diff --git a/source/common/api/win32/os_sys_calls_impl.cc b/source/common/api/win32/os_sys_calls_impl.cc index 22bd2d60d72b..86519612a253 100644 --- a/source/common/api/win32/os_sys_calls_impl.cc +++ b/source/common/api/win32/os_sys_calls_impl.cc @@ -175,6 +175,11 @@ bool OsSysCallsImpl::supportsUdpGro() const { return false; } +bool OsSysCallsImpl::supportsUdpGso() const { + // Windows doesn't support it. + return false; +} + SysCallIntResult OsSysCallsImpl::ftruncate(int fd, off_t length) { const int rc = ::_chsize_s(fd, length); return {rc, rc == 0 ? 0 : errno}; diff --git a/source/common/api/win32/os_sys_calls_impl.h b/source/common/api/win32/os_sys_calls_impl.h index d82e156de6b9..3a2ca378d658 100644 --- a/source/common/api/win32/os_sys_calls_impl.h +++ b/source/common/api/win32/os_sys_calls_impl.h @@ -23,6 +23,7 @@ class OsSysCallsImpl : public OsSysCalls { struct timespec* timeout) override; bool supportsMmsg() const override; bool supportsUdpGro() const override; + bool supportsUdpGso() const override; SysCallIntResult close(os_fd_t fd) override; SysCallIntResult ftruncate(int fd, off_t length) override; SysCallPtrResult mmap(void* addr, size_t length, int prot, int flags, int fd, diff --git a/source/common/network/BUILD b/source/common/network/BUILD index dd8bcb546337..b40195b288b7 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -235,6 +235,7 @@ envoy_cc_library( deps = [ ":address_lib", ":listen_socket_lib", + ":udp_default_writer_config", "//include/envoy/event:dispatcher_interface", "//include/envoy/event:file_event_interface", "//include/envoy/network:exception_interface", @@ -405,6 +406,32 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "udp_packet_writer_handler_lib", + srcs = ["udp_packet_writer_handler_impl.cc"], + hdrs = ["udp_packet_writer_handler_impl.h"], + deps = [ + ":io_socket_error_lib", + ":utility_lib", + "//include/envoy/network:socket_interface", + "//include/envoy/network:udp_packet_writer_config_interface", + "//include/envoy/network:udp_packet_writer_handler_interface", + "//source/common/buffer:buffer_lib", + ], +) + +envoy_cc_library( + name = "udp_default_writer_config", + srcs = ["udp_default_writer_config.cc"], + hdrs = ["udp_default_writer_config.h"], + deps = [ + ":udp_packet_writer_handler_lib", + "//include/envoy/network:udp_packet_writer_config_interface", + "//include/envoy/registry", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "proxy_protocol_filter_state_lib", srcs = ["proxy_protocol_filter_state.cc"], diff --git a/source/common/network/udp_default_writer_config.cc b/source/common/network/udp_default_writer_config.cc new file mode 100644 index 000000000000..c07336c513a8 --- /dev/null +++ b/source/common/network/udp_default_writer_config.cc @@ -0,0 +1,32 @@ +#include "common/network/udp_default_writer_config.h" + +#include +#include + +#include "envoy/config/listener/v3/udp_default_writer_config.pb.h" + +#include "common/network/udp_packet_writer_handler_impl.h" + +namespace Envoy { +namespace Network { + +UdpPacketWriterPtr UdpDefaultWriterFactory::createUdpPacketWriter(Network::IoHandle& io_handle, + Stats::Scope& /*scope*/) { + return std::make_unique(io_handle); +} + +ProtobufTypes::MessagePtr UdpDefaultWriterConfigFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +UdpPacketWriterFactoryPtr +UdpDefaultWriterConfigFactory::createUdpPacketWriterFactory(const Protobuf::Message& /*message*/) { + return std::make_unique(); +} + +std::string UdpDefaultWriterConfigFactory::name() const { return "udp_default_writer"; } + +REGISTER_FACTORY(UdpDefaultWriterConfigFactory, Network::UdpPacketWriterConfigFactory); + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/udp_default_writer_config.h b/source/common/network/udp_default_writer_config.h new file mode 100644 index 000000000000..e01c465e904f --- /dev/null +++ b/source/common/network/udp_default_writer_config.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/network/udp_packet_writer_config.h" +#include "envoy/network/udp_packet_writer_handler.h" +#include "envoy/registry/registry.h" + +namespace Envoy { +namespace Network { + +class UdpDefaultWriterFactory : public Network::UdpPacketWriterFactory { +public: + Network::UdpPacketWriterPtr createUdpPacketWriter(Network::IoHandle& io_handle, + Stats::Scope& scope) override; +}; + +// UdpPacketWriterConfigFactory to create UdpDefaultWriterFactory based on given protobuf +// This is the default UdpPacketWriterConfigFactory if not specified in config. +class UdpDefaultWriterConfigFactory : public UdpPacketWriterConfigFactory { +public: + // UdpPacketWriterConfigFactory + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + + Network::UdpPacketWriterFactoryPtr + createUdpPacketWriterFactory(const Protobuf::Message&) override; + + std::string name() const override; +}; + +DECLARE_FACTORY(UdpDefaultWriterConfigFactory); + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index 2b28e0ea5841..3eaf0272e940 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -110,8 +110,9 @@ const Address::InstanceConstSharedPtr& UdpListenerImpl::localAddress() const { Api::IoCallUint64Result UdpListenerImpl::send(const UdpSendData& send_data) { ENVOY_UDP_LOG(trace, "send"); Buffer::Instance& buffer = send_data.buffer_; - Api::IoCallUint64Result send_result = Utility::writeToSocket( - socket_->ioHandle(), buffer, send_data.local_ip_, send_data.peer_address_); + + Api::IoCallUint64Result send_result = + cb_.udpPacketWriter().writePacket(buffer, send_data.local_ip_, send_data.peer_address_); // The send_result normalizes the rc_ value to 0 in error conditions. // The drain call is hence 'safe' in success and failure cases. @@ -119,5 +120,10 @@ Api::IoCallUint64Result UdpListenerImpl::send(const UdpSendData& send_data) { return send_result; } +Api::IoCallUint64Result UdpListenerImpl::flush() { + ENVOY_UDP_LOG(trace, "flush"); + return cb_.udpPacketWriter().flush(); +} + } // namespace Network } // namespace Envoy diff --git a/source/common/network/udp_listener_impl.h b/source/common/network/udp_listener_impl.h index 9789301361c3..67168fb1c7ee 100644 --- a/source/common/network/udp_listener_impl.h +++ b/source/common/network/udp_listener_impl.h @@ -35,6 +35,7 @@ class UdpListenerImpl : public BaseListenerImpl, Event::Dispatcher& dispatcher() override; const Address::InstanceConstSharedPtr& localAddress() const override; Api::IoCallUint64Result send(const UdpSendData& data) override; + Api::IoCallUint64Result flush() override; void processPacket(Address::InstanceConstSharedPtr local_address, Address::InstanceConstSharedPtr peer_address, Buffer::InstancePtr buffer, diff --git a/source/common/network/udp_packet_writer_handler_impl.cc b/source/common/network/udp_packet_writer_handler_impl.cc new file mode 100644 index 000000000000..27d499268e28 --- /dev/null +++ b/source/common/network/udp_packet_writer_handler_impl.cc @@ -0,0 +1,28 @@ +#include "common/network/udp_packet_writer_handler_impl.h" + +#include "common/buffer/buffer_impl.h" +#include "common/network/utility.h" + +namespace Envoy { +namespace Network { + +UdpDefaultWriter::UdpDefaultWriter(Network::IoHandle& io_handle) + : write_blocked_(false), io_handle_(io_handle) {} + +UdpDefaultWriter::~UdpDefaultWriter() = default; + +Api::IoCallUint64Result UdpDefaultWriter::writePacket(const Buffer::Instance& buffer, + const Address::Ip* local_ip, + const Address::Instance& peer_address) { + ASSERT(!write_blocked_, "Cannot write while IO handle is blocked."); + Api::IoCallUint64Result result = + Utility::writeToSocket(io_handle_, buffer, local_ip, peer_address); + if (result.err_ && result.err_->getErrorCode() == Api::IoError::IoErrorCode::Again) { + // Writer is blocked when error code received is EWOULDBLOCK/EAGAIN + write_blocked_ = true; + } + return result; +} + +} // namespace Network +} // namespace Envoy diff --git a/source/common/network/udp_packet_writer_handler_impl.h b/source/common/network/udp_packet_writer_handler_impl.h new file mode 100644 index 000000000000..50c3f34b79cd --- /dev/null +++ b/source/common/network/udp_packet_writer_handler_impl.h @@ -0,0 +1,45 @@ +#pragma once + +#include "envoy/buffer/buffer.h" +#include "envoy/network/socket.h" +#include "envoy/network/udp_packet_writer_handler.h" + +#include "common/network/io_socket_error_impl.h" + +namespace Envoy { +namespace Network { + +class UdpDefaultWriter : public UdpPacketWriter { +public: + UdpDefaultWriter(Network::IoHandle& io_handle); + + ~UdpDefaultWriter() override; + + // Following writePacket utilizes Utility::writeToSocket() implementation + Api::IoCallUint64Result writePacket(const Buffer::Instance& buffer, const Address::Ip* local_ip, + const Address::Instance& peer_address) override; + + bool isWriteBlocked() const override { return write_blocked_; } + void setWritable() override { write_blocked_ = false; } + uint64_t getMaxPacketSize(const Address::Instance& /*peer_address*/) const override { + return Network::UdpMaxOutgoingPacketSize; + } + bool isBatchMode() const override { return false; } + Network::UdpPacketWriterBuffer + getNextWriteLocation(const Address::Ip* /*local_ip*/, + const Address::Instance& /*peer_address*/) override { + return {nullptr, 0, nullptr}; + } + Api::IoCallUint64Result flush() override { + return Api::IoCallUint64Result( + /*rc=*/0, + /*err=*/Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); + } + +private: + bool write_blocked_; + Network::IoHandle& io_handle_; +}; + +} // namespace Network +} // namespace Envoy diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index 8afb8035dbf5..095bc869f7e6 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -206,6 +206,8 @@ void UdpProxyFilter::ActiveSession::onReadReady() { if (result->getErrorCode() != Api::IoError::IoErrorCode::Again) { cluster_.cluster_stats_.sess_rx_errors_.inc(); } + // Flush out buffered data at the end of IO event. + cluster_.filter_.read_callbacks_->udpListener().flush(); } void UdpProxyFilter::ActiveSession::write(const Buffer::Instance& buffer) { diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index fd2cce9b9b5f..31a4ff5dec98 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -49,18 +49,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "envoy_quic_packet_writer_lib", - srcs = ["envoy_quic_packet_writer.cc"], - hdrs = ["envoy_quic_packet_writer.h"], - external_deps = ["quiche_quic_platform"], - tags = ["nofips"], - deps = [ - ":envoy_quic_utils_lib", - "@com_googlesource_quiche//:quic_core_packet_writer_interface_lib", - ], -) - envoy_cc_library( name = "envoy_quic_proof_source_base_lib", srcs = ["envoy_quic_proof_source_base.cc"], @@ -274,6 +262,7 @@ envoy_cc_library( ":envoy_quic_packet_writer_lib", "//include/envoy/event:dispatcher_interface", "//source/common/network:socket_option_factory_lib", + "//source/common/network:udp_packet_writer_handler_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) @@ -313,6 +302,7 @@ envoy_cc_library( ":envoy_quic_packet_writer_lib", ":envoy_quic_proof_source_lib", ":envoy_quic_utils_lib", + ":udp_gso_batch_writer_lib", "//include/envoy/network:listener_interface", "//source/common/network:listener_lib", "//source/common/protobuf:utility_lib", @@ -377,6 +367,55 @@ envoy_cc_extension( ], ) +envoy_cc_library( + name = "envoy_quic_packet_writer_lib", + srcs = ["envoy_quic_packet_writer.cc"], + hdrs = ["envoy_quic_packet_writer.h"], + external_deps = ["quiche_quic_platform"], + tags = ["nofips"], + deps = [ + ":envoy_quic_utils_lib", + "@com_googlesource_quiche//:quic_core_packet_writer_interface_lib", + ], +) + +envoy_cc_library( + name = "udp_gso_batch_writer_lib", + srcs = ["udp_gso_batch_writer.cc"], + hdrs = ["udp_gso_batch_writer.h"], + external_deps = ["quiche_quic_platform"], + tags = ["nofips"], + visibility = [ + "//test/common/network:__subpackages__", + "//test/extensions:__subpackages__", + ], + deps = [ + ":envoy_quic_utils_lib", + "//include/envoy/network:udp_packet_writer_handler_interface", + "//source/common/network:io_socket_error_lib", + "//source/common/protobuf:utility_lib", + "//source/common/runtime:runtime_lib", + "@com_googlesource_quiche//:quic_core_batch_writer_gso_batch_writer_lib", + ], +) + +envoy_cc_library( + name = "udp_gso_batch_writer_config_lib", + srcs = ["udp_gso_batch_writer_config.cc"], + hdrs = ["udp_gso_batch_writer_config.h"], + tags = ["nofips"], + visibility = [ + "//test/server:__subpackages__", + ], + deps = [ + ":udp_gso_batch_writer_lib", + "//include/envoy/network:udp_packet_writer_config_interface", + "//include/envoy/registry", + "//source/common/api:os_sys_calls_lib", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "envoy_quic_crypto_server_stream_lib", srcs = ["envoy_quic_crypto_server_stream.cc"], diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.cc b/source/extensions/quic_listeners/quiche/active_quic_listener.cc index c691e39a5551..eda0b7210e72 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.cc +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.cc @@ -12,8 +12,9 @@ #include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" #include "extensions/quic_listeners/quiche/envoy_quic_dispatcher.h" #include "extensions/quic_listeners/quiche/envoy_quic_proof_source.h" -#include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" +#include "extensions/quic_listeners/quiche/udp_gso_batch_writer.h" namespace Envoy { namespace Quic { @@ -66,7 +67,20 @@ ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, crypto_config_.get(), quic_config, &version_manager_, std::move(connection_helper), std::move(alarm_factory), quic::kQuicDefaultConnectionIdLength, parent, *config_, stats_, per_worker_stats_, dispatcher, listen_socket_); - quic_dispatcher_->InitializeWithWriter(new EnvoyQuicPacketWriter(listen_socket_)); + + // Create udp_packet_writer + Network::UdpPacketWriterPtr udp_packet_writer = + listener_config.udpPacketWriterFactory()->get().createUdpPacketWriter( + listen_socket_.ioHandle(), listener_config.listenerScope()); + udp_packet_writer_ = udp_packet_writer.get(); + if (udp_packet_writer->isBatchMode()) { + // UdpPacketWriter* can be downcasted to UdpGsoBatchWriter*, which indirectly inherits + // from the quic::QuicPacketWriter class and can be passed to InitializeWithWriter(). + quic_dispatcher_->InitializeWithWriter( + dynamic_cast(udp_packet_writer.release())); + } else { + quic_dispatcher_->InitializeWithWriter(new EnvoyQuicPacketWriter(std::move(udp_packet_writer))); + } } ActiveQuicListener::~ActiveQuicListener() { onListenerShutdown(); } @@ -79,9 +93,9 @@ void ActiveQuicListener::onListenerShutdown() { void ActiveQuicListener::onData(Network::UdpRecvData& data) { quic::QuicSocketAddress peer_address( - envoyAddressInstanceToQuicSocketAddress(data.addresses_.peer_)); + envoyIpAddressToQuicSocketAddress(data.addresses_.peer_->ip())); quic::QuicSocketAddress self_address( - envoyAddressInstanceToQuicSocketAddress(data.addresses_.local_)); + envoyIpAddressToQuicSocketAddress(data.addresses_.local_->ip())); quic::QuicTime timestamp = quic::QuicTime::Zero() + quic::QuicTime::Delta::FromMicroseconds(std::chrono::duration_cast( diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.h b/source/extensions/quic_listeners/quiche/active_quic_listener.h index 8d0d5c9dd46e..08b7807dfc4f 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.h +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.h @@ -47,6 +47,7 @@ class ActiveQuicListener : public Network::UdpListenerCallbacks, void onReceiveError(Api::IoError::IoErrorCode /*error_code*/) override { // No-op. Quic can't do anything upon listener error. } + Network::UdpPacketWriter& udpPacketWriter() override { return *udp_packet_writer_; } // ActiveListenerImplBase Network::Listener* listener() override { return udp_listener_.get(); } @@ -65,6 +66,7 @@ class ActiveQuicListener : public Network::UdpListenerCallbacks, std::unique_ptr quic_dispatcher_; Network::Socket& listen_socket_; Runtime::FeatureFlag enabled_; + Network::UdpPacketWriter* udp_packet_writer_; }; using ActiveQuicListenerPtr = std::unique_ptr; diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_client_connection.cc b/source/extensions/quic_listeners/quiche/envoy_quic_client_connection.cc index c8cfe4d14d69..bb3c172536df 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_client_connection.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_client_connection.cc @@ -6,6 +6,7 @@ #include "common/network/listen_socket_impl.h" #include "common/network/socket_option_factory.h" +#include "common/network/udp_packet_writer_handler_impl.h" #include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" @@ -30,9 +31,11 @@ EnvoyQuicClientConnection::EnvoyQuicClientConnection( const quic::QuicConnectionId& server_connection_id, quic::QuicConnectionHelperInterface& helper, quic::QuicAlarmFactory& alarm_factory, const quic::ParsedQuicVersionVector& supported_versions, Event::Dispatcher& dispatcher, Network::ConnectionSocketPtr&& connection_socket) - : EnvoyQuicClientConnection(server_connection_id, helper, alarm_factory, - new EnvoyQuicPacketWriter(*connection_socket), true, - supported_versions, dispatcher, std::move(connection_socket)) {} + : EnvoyQuicClientConnection( + server_connection_id, helper, alarm_factory, + new EnvoyQuicPacketWriter( + std::make_unique(connection_socket->ioHandle())), + true, supported_versions, dispatcher, std::move(connection_socket)) {} EnvoyQuicClientConnection::EnvoyQuicClientConnection( const quic::QuicConnectionId& server_connection_id, quic::QuicConnectionHelperInterface& helper, @@ -41,7 +44,7 @@ EnvoyQuicClientConnection::EnvoyQuicClientConnection( Network::ConnectionSocketPtr&& connection_socket) : EnvoyQuicConnection( server_connection_id, - envoyAddressInstanceToQuicSocketAddress(connection_socket->remoteAddress()), helper, + envoyIpAddressToQuicSocketAddress(connection_socket->remoteAddress()->ip()), helper, alarm_factory, writer, owns_writer, quic::Perspective::IS_CLIENT, supported_versions, std::move(connection_socket)), dispatcher_(dispatcher) {} @@ -64,8 +67,8 @@ void EnvoyQuicClientConnection::processPacket( timestamp, /*owns_buffer=*/false, /*ttl=*/0, /*ttl_valid=*/false, /*packet_headers=*/nullptr, /*headers_length=*/0, /*owns_header_buffer*/ false); - ProcessUdpPacket(envoyAddressInstanceToQuicSocketAddress(local_address), - envoyAddressInstanceToQuicSocketAddress(peer_address), packet); + ProcessUdpPacket(envoyIpAddressToQuicSocketAddress(local_address->ip()), + envoyIpAddressToQuicSocketAddress(peer_address->ip()), packet); } uint64_t EnvoyQuicClientConnection::maxPacketSize() const { @@ -94,7 +97,8 @@ void EnvoyQuicClientConnection::setUpConnectionSocket() { void EnvoyQuicClientConnection::switchConnectionSocket( Network::ConnectionSocketPtr&& connection_socket) { - auto writer = std::make_unique(*connection_socket); + auto writer = std::make_unique( + std::make_unique(connection_socket->ioHandle())); // Destroy the old file_event before closing the old socket. Otherwise the socket might be picked // up by another socket() call while file_event is still operating on it. file_event_.reset(); diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc index 88816a34d059..a6a70623a43f 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc @@ -1,43 +1,75 @@ #include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" -#include "common/buffer/buffer_impl.h" -#include "common/network/utility.h" +#include #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" namespace Envoy { namespace Quic { -EnvoyQuicPacketWriter::EnvoyQuicPacketWriter(Network::Socket& socket) - : write_blocked_(false), socket_(socket) {} -quic::WriteResult EnvoyQuicPacketWriter::WritePacket(const char* buffer, size_t buf_len, +namespace { + +quic::WriteResult convertToQuicWriteResult(Api::IoCallUint64Result& result) { + if (result.ok()) { + return {quic::WRITE_STATUS_OK, static_cast(result.rc_)}; + } + quic::WriteStatus status = result.err_->getErrorCode() == Api::IoError::IoErrorCode::Again + ? quic::WRITE_STATUS_BLOCKED + : quic::WRITE_STATUS_ERROR; + return {status, static_cast(result.err_->getErrorCode())}; +} + +} // namespace + +EnvoyQuicPacketWriter::EnvoyQuicPacketWriter(Network::UdpPacketWriterPtr envoy_udp_packet_writer) + : envoy_udp_packet_writer_(std::move(envoy_udp_packet_writer)) {} + +quic::WriteResult EnvoyQuicPacketWriter::WritePacket(const char* buffer, size_t buffer_len, const quic::QuicIpAddress& self_ip, const quic::QuicSocketAddress& peer_address, quic::PerPacketOptions* options) { ASSERT(options == nullptr, "Per packet option is not supported yet."); - ASSERT(!write_blocked_, "Cannot write while IO handle is blocked."); - Buffer::RawSlice slice; - slice.mem_ = const_cast(buffer); - slice.len_ = buf_len; + Buffer::BufferFragmentImpl fragment(buffer, buffer_len, nullptr); + Buffer::OwnedImpl buf; + buf.addBufferFragment(fragment); + quic::QuicSocketAddress self_address(self_ip, /*port=*/0); Network::Address::InstanceConstSharedPtr local_addr = quicAddressToEnvoyAddressInstance(self_address); Network::Address::InstanceConstSharedPtr remote_addr = quicAddressToEnvoyAddressInstance(peer_address); - Api::IoCallUint64Result result = Network::Utility::writeToSocket( - socket_.ioHandle(), &slice, 1, local_addr == nullptr ? nullptr : local_addr->ip(), - *remote_addr); - if (result.ok()) { - return {quic::WRITE_STATUS_OK, static_cast(result.rc_)}; - } - quic::WriteStatus status = result.err_->getErrorCode() == Api::IoError::IoErrorCode::Again - ? quic::WRITE_STATUS_BLOCKED - : quic::WRITE_STATUS_ERROR; - if (quic::IsWriteBlockedStatus(status)) { - write_blocked_ = true; - } - return {status, static_cast(result.err_->getErrorCode())}; + + Api::IoCallUint64Result result = envoy_udp_packet_writer_->writePacket( + buf, local_addr == nullptr ? nullptr : local_addr->ip(), *remote_addr); + + return convertToQuicWriteResult(result); +} + +quic::QuicByteCount +EnvoyQuicPacketWriter::GetMaxPacketSize(const quic::QuicSocketAddress& peer_address) const { + Network::Address::InstanceConstSharedPtr remote_addr = + quicAddressToEnvoyAddressInstance(peer_address); + return static_cast(envoy_udp_packet_writer_->getMaxPacketSize(*remote_addr)); +} + +quic::QuicPacketBuffer +EnvoyQuicPacketWriter::GetNextWriteLocation(const quic::QuicIpAddress& self_ip, + const quic::QuicSocketAddress& peer_address) { + quic::QuicSocketAddress self_address(self_ip, /*port=*/0); + Network::Address::InstanceConstSharedPtr local_addr = + quicAddressToEnvoyAddressInstance(self_address); + Network::Address::InstanceConstSharedPtr remote_addr = + quicAddressToEnvoyAddressInstance(peer_address); + Network::UdpPacketWriterBuffer write_location = envoy_udp_packet_writer_->getNextWriteLocation( + local_addr == nullptr ? nullptr : local_addr->ip(), *remote_addr); + return quic::QuicPacketBuffer(reinterpret_cast(write_location.buffer_), + write_location.release_buffer_); +} + +quic::WriteResult EnvoyQuicPacketWriter::Flush() { + Api::IoCallUint64Result result = envoy_udp_packet_writer_->flush(); + return convertToQuicWriteResult(result); } } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h index 4d2eed570165..bb4b736c84c8 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h @@ -10,14 +10,14 @@ #pragma GCC diagnostic pop -#include "envoy/network/listener.h" +#include "envoy/network/udp_packet_writer_handler.h" namespace Envoy { namespace Quic { class EnvoyQuicPacketWriter : public quic::QuicPacketWriter { public: - EnvoyQuicPacketWriter(Network::Socket& socket); + EnvoyQuicPacketWriter(Network::UdpPacketWriterPtr envoy_udp_packet_writer); quic::WriteResult WritePacket(const char* buffer, size_t buf_len, const quic::QuicIpAddress& self_address, @@ -25,26 +25,19 @@ class EnvoyQuicPacketWriter : public quic::QuicPacketWriter { quic::PerPacketOptions* options) override; // quic::QuicPacketWriter - bool IsWriteBlocked() const override { return write_blocked_; } - void SetWritable() override { write_blocked_ = false; } - quic::QuicByteCount - GetMaxPacketSize(const quic::QuicSocketAddress& /*peer_address*/) const override { - return quic::kMaxOutgoingPacketSize; - } - // Currently this writer doesn't support pacing offload or batch writing. + bool IsWriteBlocked() const override { return envoy_udp_packet_writer_->isWriteBlocked(); } + void SetWritable() override { envoy_udp_packet_writer_->setWritable(); } + bool IsBatchMode() const override { return envoy_udp_packet_writer_->isBatchMode(); } + // Currently this writer doesn't support pacing offload. bool SupportsReleaseTime() const override { return false; } - bool IsBatchMode() const override { return false; } - quic::QuicPacketBuffer - GetNextWriteLocation(const quic::QuicIpAddress& /*self_address*/, - const quic::QuicSocketAddress& /*peer_address*/) override { - return {nullptr, nullptr}; - } - quic::WriteResult Flush() override { return {quic::WRITE_STATUS_OK, 0}; } + + quic::QuicByteCount GetMaxPacketSize(const quic::QuicSocketAddress& peer_address) const override; + quic::QuicPacketBuffer GetNextWriteLocation(const quic::QuicIpAddress& self_address, + const quic::QuicSocketAddress& peer_address) override; + quic::WriteResult Flush() override; private: - // Modified by WritePacket() to indicate underlying IoHandle status. - bool write_blocked_; - Network::Socket& socket_; + Network::UdpPacketWriterPtr envoy_udp_packet_writer_; }; } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc b/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc index b5c710a81269..c7a32fbf317d 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc @@ -22,25 +22,30 @@ quicAddressToEnvoyAddressInstance(const quic::QuicSocketAddress& quic_address) { : nullptr; } -quic::QuicSocketAddress envoyAddressInstanceToQuicSocketAddress( - const Network::Address::InstanceConstSharedPtr& envoy_address) { - ASSERT(envoy_address != nullptr && envoy_address->type() == Network::Address::Type::Ip); - uint32_t port = envoy_address->ip()->port(); +quic::QuicSocketAddress envoyIpAddressToQuicSocketAddress(const Network::Address::Ip* envoy_ip) { + if (envoy_ip == nullptr) { + // Return uninitialized socket addr + return quic::QuicSocketAddress(); + } + + uint32_t port = envoy_ip->port(); sockaddr_storage ss; - if (envoy_address->ip()->version() == Network::Address::IpVersion::v4) { + + if (envoy_ip->version() == Network::Address::IpVersion::v4) { + // Create and return quic ipv4 address auto ipv4_addr = reinterpret_cast(&ss); memset(ipv4_addr, 0, sizeof(sockaddr_in)); ipv4_addr->sin_family = AF_INET; ipv4_addr->sin_port = htons(port); - ipv4_addr->sin_addr.s_addr = envoy_address->ip()->ipv4()->address(); + ipv4_addr->sin_addr.s_addr = envoy_ip->ipv4()->address(); } else { + // Create and return quic ipv6 address auto ipv6_addr = reinterpret_cast(&ss); memset(ipv6_addr, 0, sizeof(sockaddr_in6)); ipv6_addr->sin6_family = AF_INET6; ipv6_addr->sin6_port = htons(port); ASSERT(sizeof(ipv6_addr->sin6_addr.s6_addr) == 16u); - *reinterpret_cast(ipv6_addr->sin6_addr.s6_addr) = - envoy_address->ip()->ipv6()->address(); + *reinterpret_cast(ipv6_addr->sin6_addr.s6_addr) = envoy_ip->ipv6()->address(); } return quic::QuicSocketAddress(ss); } diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_utils.h b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h index 34dce87d836b..5c321ab749f1 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_utils.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h @@ -34,8 +34,7 @@ namespace Quic { Network::Address::InstanceConstSharedPtr quicAddressToEnvoyAddressInstance(const quic::QuicSocketAddress& quic_address); -quic::QuicSocketAddress envoyAddressInstanceToQuicSocketAddress( - const Network::Address::InstanceConstSharedPtr& envoy_address); +quic::QuicSocketAddress envoyIpAddressToQuicSocketAddress(const Network::Address::Ip* envoy_ip); // The returned header map has all keys in lower case. template diff --git a/source/extensions/quic_listeners/quiche/udp_gso_batch_writer.cc b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer.cc new file mode 100644 index 000000000000..5525ee285d41 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer.cc @@ -0,0 +1,126 @@ +#include "extensions/quic_listeners/quiche/udp_gso_batch_writer.h" + +#include "common/network/io_socket_error_impl.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" + +namespace Envoy { +namespace Quic { +namespace { +Api::IoCallUint64Result convertQuicWriteResult(quic::WriteResult quic_result, size_t payload_len) { + switch (quic_result.status) { + case quic::WRITE_STATUS_OK: { + if (quic_result.bytes_written == 0) { + ENVOY_LOG_MISC(trace, "sendmsg successful, message buffered to send"); + } else { + ENVOY_LOG_MISC(trace, "sendmsg successful, flushed bytes {}", quic_result.bytes_written); + } + // Return payload_len as rc & nullptr as error on success + return Api::IoCallUint64Result( + /*rc=*/payload_len, + /*err=*/Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); + } + case quic::WRITE_STATUS_BLOCKED_DATA_BUFFERED: { + // Data was buffered, Return payload_len as rc & nullptr as error + ENVOY_LOG_MISC(trace, "sendmsg blocked, message buffered to send"); + return Api::IoCallUint64Result( + /*rc=*/payload_len, + /*err=*/Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); + } + case quic::WRITE_STATUS_BLOCKED: { + // Writer blocked, return error + ENVOY_LOG_MISC(trace, "sendmsg blocked, message not buffered"); + return Api::IoCallUint64Result( + /*rc=*/0, + /*err=*/Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), + Network::IoSocketError::deleteIoError)); + } + default: { + // Write Failed, return {0 and error_code} + ENVOY_LOG_MISC(trace, "sendmsg failed with error code {}", + static_cast(quic_result.error_code)); + return Api::IoCallUint64Result( + /*rc=*/0, + /*err=*/Api::IoErrorPtr(new Network::IoSocketError(quic_result.error_code), + Network::IoSocketError::deleteIoError)); + } + } +} + +} // namespace + +// Initialize QuicGsoBatchWriter, set io_handle_ and stats_ +UdpGsoBatchWriter::UdpGsoBatchWriter(Network::IoHandle& io_handle, Stats::Scope& scope) + : quic::QuicGsoBatchWriter(std::make_unique(), io_handle.fd()), + stats_(generateStats(scope)) {} + +// Do Nothing in the Destructor For now +UdpGsoBatchWriter::~UdpGsoBatchWriter() = default; + +Api::IoCallUint64Result +UdpGsoBatchWriter::writePacket(const Buffer::Instance& buffer, const Network::Address::Ip* local_ip, + const Network::Address::Instance& peer_address) { + // Convert received parameters to relevant forms + quic::QuicSocketAddress peer_addr = envoyIpAddressToQuicSocketAddress(peer_address.ip()); + quic::QuicSocketAddress self_addr = envoyIpAddressToQuicSocketAddress(local_ip); + size_t payload_len = static_cast(buffer.length()); + + // TODO(yugant): Currently we do not use PerPacketOptions with Quic, we may want to + // specify this parameter here at a later stage. + quic::WriteResult quic_result = + WritePacket(buffer.toString().c_str(), payload_len, self_addr.host(), peer_addr, + /*quic::PerPacketOptions=*/nullptr); + updateUdpGsoBatchWriterStats(quic_result); + + return convertQuicWriteResult(quic_result, payload_len); +} + +uint64_t UdpGsoBatchWriter::getMaxPacketSize(const Network::Address::Instance& peer_address) const { + quic::QuicSocketAddress peer_addr = envoyIpAddressToQuicSocketAddress(peer_address.ip()); + return static_cast(GetMaxPacketSize(peer_addr)); +} + +Network::UdpPacketWriterBuffer +UdpGsoBatchWriter::getNextWriteLocation(const Network::Address::Ip* local_ip, + const Network::Address::Instance& peer_address) { + quic::QuicSocketAddress peer_addr = envoyIpAddressToQuicSocketAddress(peer_address.ip()); + quic::QuicSocketAddress self_addr = envoyIpAddressToQuicSocketAddress(local_ip); + quic::QuicPacketBuffer quic_buf = GetNextWriteLocation(self_addr.host(), peer_addr); + return Network::UdpPacketWriterBuffer(reinterpret_cast(quic_buf.buffer), + Network::UdpMaxOutgoingPacketSize, quic_buf.release_buffer); +} + +Api::IoCallUint64Result UdpGsoBatchWriter::flush() { + quic::WriteResult quic_result = Flush(); + updateUdpGsoBatchWriterStats(quic_result); + + return convertQuicWriteResult(quic_result, /*payload_len=*/0); +} + +void UdpGsoBatchWriter::updateUdpGsoBatchWriterStats(quic::WriteResult quic_result) { + if (quic_result.status == quic::WRITE_STATUS_OK && quic_result.bytes_written > 0) { + if (gso_size_ > 0u) { + uint64_t num_pkts_in_batch = + std::ceil(static_cast(quic_result.bytes_written) / gso_size_); + stats_.pkts_sent_per_batch_.recordValue(num_pkts_in_batch); + } + stats_.total_bytes_sent_.add(quic_result.bytes_written); + } + stats_.internal_buffer_size_.set(batch_buffer().SizeInUse()); + gso_size_ = buffered_writes().empty() ? 0u : buffered_writes().front().buf_len; +} + +UdpGsoBatchWriterStats UdpGsoBatchWriter::generateStats(Stats::Scope& scope) { + return { + UDP_GSO_BATCH_WRITER_STATS(POOL_COUNTER(scope), POOL_GAUGE(scope), POOL_HISTOGRAM(scope))}; +} + +UdpGsoBatchWriterFactory::UdpGsoBatchWriterFactory() = default; + +Network::UdpPacketWriterPtr +UdpGsoBatchWriterFactory::createUdpPacketWriter(Network::IoHandle& io_handle, Stats::Scope& scope) { + return std::make_unique(io_handle, scope); +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/udp_gso_batch_writer.h b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer.h new file mode 100644 index 000000000000..477ad8bdcdc7 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer.h @@ -0,0 +1,124 @@ +#pragma once + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +// QUICHE allows ignored qualifiers +#pragma GCC diagnostic ignored "-Wignored-qualifiers" + +// QUICHE doesn't mark override at QuicBatchWriterBase::SupportsReleaseTime() +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Winconsistent-missing-override" +#elif defined(__GNUC__) && __GNUC__ >= 5 +#pragma GCC diagnostic ignored "-Wsuggest-override" +#endif + +#include "quiche/quic/core/batch_writer/quic_gso_batch_writer.h" + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +#pragma GCC diagnostic pop + +#include "envoy/network/udp_packet_writer_handler.h" + +#include "common/protobuf/utility.h" +#include "common/runtime/runtime_protos.h" + +namespace Envoy { +namespace Quic { + +/** + * @brief The following can be used to collect statistics + * related to UdpGsoBatchWriter. The stats maintained are + * as follows: + * + * @total_bytes_sent: Maintains the count of total bytes + * sent via the UdpGsoBatchWriter on the current ioHandle + * via both WritePacket() and Flush() functions. + * + * @internal_buffer_size: Gauge value to keep a track of the + * total bytes buffered to writer by UdpGsoBatchWriter. + * Resets whenever the internal bytes are sent to the client. + * + * @pkts_sent_per_batch: Histogram to keep maintain stats of + * total number of packets sent in each batch by UdpGsoBatchWriter + * Provides summary count of batch-sizes within bucketed range, + * and also provides sum and count stats. + * + * TODO(danzh): Add writer stats to QUIC Documentation when it is + * created for QUIC/HTTP3 docs. Also specify in the documentation + * that user has to compile in QUICHE to use UdpGsoBatchWriter. + */ +#define UDP_GSO_BATCH_WRITER_STATS(COUNTER, GAUGE, HISTOGRAM) \ + COUNTER(total_bytes_sent) \ + GAUGE(internal_buffer_size, NeverImport) \ + HISTOGRAM(pkts_sent_per_batch, Unspecified) + +/** + * Wrapper struct for udp gso batch writer stats. @see stats_macros.h + */ +struct UdpGsoBatchWriterStats { + UDP_GSO_BATCH_WRITER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, + GENERATE_HISTOGRAM_STRUCT) +}; + +/** + * UdpPacketWriter implementation based on quic::QuicGsoBatchWriter to send packets + * in batches, using UDP socket's generic segmentation offload(GSO) capability. + */ +class UdpGsoBatchWriter : public quic::QuicGsoBatchWriter, public Network::UdpPacketWriter { +public: + UdpGsoBatchWriter(Network::IoHandle& io_handle, Stats::Scope& scope); + + ~UdpGsoBatchWriter() override; + + // writePacket perform batched sends based on QuicGsoBatchWriter::WritePacket + Api::IoCallUint64Result writePacket(const Buffer::Instance& buffer, + const Network::Address::Ip* local_ip, + const Network::Address::Instance& peer_address) override; + + // UdpPacketWriter Implementations + bool isWriteBlocked() const override { return IsWriteBlocked(); } + void setWritable() override { return SetWritable(); } + bool isBatchMode() const override { return IsBatchMode(); } + uint64_t getMaxPacketSize(const Network::Address::Instance& peer_address) const override; + Network::UdpPacketWriterBuffer + getNextWriteLocation(const Network::Address::Ip* local_ip, + const Network::Address::Instance& peer_address) override; + Api::IoCallUint64Result flush() override; + +private: + /** + * @brief Update stats_ field for the udp packet writer + * @param quic_result is the result from Flush/WritePacket + */ + void updateUdpGsoBatchWriterStats(quic::WriteResult quic_result); + + /** + * @brief Generate UdpGsoBatchWriterStats object from scope + * @param scope for stats + * @return UdpGsoBatchWriterStats for scope + */ + UdpGsoBatchWriterStats generateStats(Stats::Scope& scope); + UdpGsoBatchWriterStats stats_; + uint64_t gso_size_; +}; + +class UdpGsoBatchWriterFactory : public Network::UdpPacketWriterFactory { +public: + UdpGsoBatchWriterFactory(); + + Network::UdpPacketWriterPtr createUdpPacketWriter(Network::IoHandle& io_handle, + Stats::Scope& scope) override; + +private: + envoy::config::core::v3::RuntimeFeatureFlag enabled_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.cc b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.cc new file mode 100644 index 000000000000..e2428f32ecaf --- /dev/null +++ b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.cc @@ -0,0 +1,30 @@ +#include "extensions/quic_listeners/quiche/udp_gso_batch_writer_config.h" + +#include "envoy/config/listener/v3/udp_gso_batch_writer_config.pb.h" + +#include "common/api/os_sys_calls_impl.h" + +#include "extensions/quic_listeners/quiche/udp_gso_batch_writer.h" + +namespace Envoy { +namespace Quic { + +ProtobufTypes::MessagePtr UdpGsoBatchWriterConfigFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +Network::UdpPacketWriterFactoryPtr +UdpGsoBatchWriterConfigFactory::createUdpPacketWriterFactory(const Protobuf::Message& /*message*/) { + if (!Api::OsSysCallsSingleton::get().supportsUdpGso()) { + throw EnvoyException("Error configuring batch writer on platform without support " + "for UDP GSO. Reset udp_writer_config to default writer"); + } + return std::make_unique(); +} + +std::string UdpGsoBatchWriterConfigFactory::name() const { return GsoBatchWriterName; } + +REGISTER_FACTORY(UdpGsoBatchWriterConfigFactory, Network::UdpPacketWriterConfigFactory); + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.h b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.h new file mode 100644 index 000000000000..20c286808872 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/udp_gso_batch_writer_config.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "envoy/network/udp_packet_writer_config.h" +#include "envoy/registry/registry.h" + +namespace Envoy { +namespace Quic { + +const std::string GsoBatchWriterName{"udp_gso_batch_writer"}; + +// Network::UdpPacketWriterConfigFactory to create UdpGsoBatchWriterFactory based on given +// protobuf. +class UdpGsoBatchWriterConfigFactory : public Network::UdpPacketWriterConfigFactory { +public: + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + + Network::UdpPacketWriterFactoryPtr + createUdpPacketWriterFactory(const Protobuf::Message&) override; + + std::string name() const override; +}; + +DECLARE_FACTORY(UdpGsoBatchWriterConfigFactory); + +} // namespace Quic +} // namespace Envoy diff --git a/source/server/BUILD b/source/server/BUILD index fb1efa139ec4..7bfcd7699576 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -305,6 +305,7 @@ envoy_cc_library( ":well_known_names_lib", "//include/envoy/access_log:access_log_interface", "//include/envoy/network:connection_interface", + "//include/envoy/network:udp_packet_writer_config_interface", "//include/envoy/server:active_udp_listener_config_interface", "//include/envoy/server:api_listener_interface", "//include/envoy/server:filter_config_interface", diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 62bfb2f80acc..e3c66660fdc5 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -379,6 +379,9 @@ class AdminImpl : public Admin, Network::ActiveUdpListenerFactory* udpListenerFactory() override { NOT_REACHED_GCOVR_EXCL_LINE; } + Network::UdpPacketWriterFactoryOptRef udpPacketWriterFactory() override { + NOT_REACHED_GCOVR_EXCL_LINE; + } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; } diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index c0becac81ad5..323ddf4df430 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -545,16 +545,30 @@ ConnectionHandlerImpl::ActiveTcpConnection::~ActiveTcpConnection() { ActiveRawUdpListener::ActiveRawUdpListener(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, Network::ListenerConfig& config) - : ActiveRawUdpListener( - parent, - dispatcher.createUdpListener(config.listenSocketFactory().getListenSocket(), *this), - config) {} + : ActiveRawUdpListener(parent, config.listenSocketFactory().getListenSocket(), dispatcher, + config) {} + +ActiveRawUdpListener::ActiveRawUdpListener(Network::ConnectionHandler& parent, + Network::SocketSharedPtr listen_socket_ptr, + Event::Dispatcher& dispatcher, + Network::ListenerConfig& config) + : ActiveRawUdpListener(parent, *listen_socket_ptr, listen_socket_ptr, dispatcher, config) {} ActiveRawUdpListener::ActiveRawUdpListener(Network::ConnectionHandler& parent, + Network::Socket& listen_socket, + Network::SocketSharedPtr listen_socket_ptr, + Event::Dispatcher& dispatcher, + Network::ListenerConfig& config) + : ActiveRawUdpListener(parent, listen_socket, + dispatcher.createUdpListener(std::move(listen_socket_ptr), *this), + config) {} + +ActiveRawUdpListener::ActiveRawUdpListener(Network::ConnectionHandler& parent, + Network::Socket& listen_socket, Network::UdpListenerPtr&& listener, Network::ListenerConfig& config) : ConnectionHandlerImpl::ActiveListenerImplBase(parent, &config), - udp_listener_(std::move(listener)), read_filter_(nullptr) { + udp_listener_(std::move(listener)), read_filter_(nullptr), listen_socket_(listen_socket) { // Create the filter chain on creating a new udp listener config_->filterChainFactory().createUdpListenerFilterChain(*this, *this); @@ -564,6 +578,10 @@ ActiveRawUdpListener::ActiveRawUdpListener(Network::ConnectionHandler& parent, fmt::format("Cannot create listener as no read filter registered for the udp listener: {} ", config_->name())); } + + // Create udp_packet_writer + udp_packet_writer_ = config.udpPacketWriterFactory()->get().createUdpPacketWriter( + listen_socket_.ioHandle(), config.listenerScope()); } void ActiveRawUdpListener::onData(Network::UdpRecvData& data) { read_filter_->onData(data); } @@ -574,6 +592,9 @@ void ActiveRawUdpListener::onWriteReady(const Network::Socket&) { // TODO(sumukhs): This is not used now. When write filters are implemented, this is a // trigger to invoke the on write ready API on the filters which is when they can write // data + + // Clear write_blocked_ status for udpPacketWriter + udp_packet_writer_->setWritable(); } void ActiveRawUdpListener::onReceiveError(Api::IoError::IoErrorCode error_code) { diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index 17c94ded87a3..63a8c97575f3 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -348,14 +348,21 @@ class ActiveRawUdpListener : public Network::UdpListenerCallbacks, public: ActiveRawUdpListener(Network::ConnectionHandler& parent, Event::Dispatcher& dispatcher, Network::ListenerConfig& config); - ActiveRawUdpListener(Network::ConnectionHandler& parent, Network::UdpListenerPtr&& listener, + ActiveRawUdpListener(Network::ConnectionHandler& parent, + Network::SocketSharedPtr listen_socket_ptr, Event::Dispatcher& dispatcher, Network::ListenerConfig& config); + ActiveRawUdpListener(Network::ConnectionHandler& parent, Network::Socket& listen_socket, + Network::SocketSharedPtr listen_socket_ptr, Event::Dispatcher& dispatcher, + Network::ListenerConfig& config); + ActiveRawUdpListener(Network::ConnectionHandler& parent, Network::Socket& listen_socket, + Network::UdpListenerPtr&& listener, Network::ListenerConfig& config); // Network::UdpListenerCallbacks void onData(Network::UdpRecvData& data) override; void onReadReady() override; void onWriteReady(const Network::Socket& socket) override; void onReceiveError(Api::IoError::IoErrorCode error_code) override; + Network::UdpPacketWriter& udpPacketWriter() override { return *udp_packet_writer_; } // ActiveListenerImplBase Network::Listener* listener() override { return udp_listener_.get(); } @@ -379,6 +386,8 @@ class ActiveRawUdpListener : public Network::UdpListenerCallbacks, private: Network::UdpListenerPtr udp_listener_; Network::UdpListenerReadFilterPtr read_filter_; + Network::UdpPacketWriterPtr udp_packet_writer_; + Network::Socket& listen_socket_; }; } // namespace Server diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index bd6c81b6c62e..f3fe37ade187 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -5,6 +5,7 @@ #include "envoy/config/listener/v3/listener_components.pb.h" #include "envoy/extensions/filters/listener/proxy_protocol/v3/proxy_protocol.pb.h" #include "envoy/network/exception.h" +#include "envoy/network/udp_packet_writer_config.h" #include "envoy/registry/registry.h" #include "envoy/server/active_udp_listener_config.h" #include "envoy/server/transport_socket_config.h" @@ -276,6 +277,7 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, auto socket_type = Network::Utility::protobufAddressSocketType(config.address()); buildListenSocketOptions(socket_type); buildUdpListenerFactory(socket_type, concurrency); + buildUdpWriterFactory(socket_type); createListenerFilterFactories(socket_type); validateFilterChains(socket_type); buildFilterChains(); @@ -331,6 +333,7 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, auto socket_type = Network::Utility::protobufAddressSocketType(config.address()); buildListenSocketOptions(socket_type); buildUdpListenerFactory(socket_type, concurrency); + buildUdpWriterFactory(socket_type); createListenerFilterFactories(socket_type); validateFilterChains(socket_type); buildFilterChains(); @@ -372,6 +375,24 @@ void ListenerImpl::buildUdpListenerFactory(Network::Socket::Type socket_type, } } +void ListenerImpl::buildUdpWriterFactory(Network::Socket::Type socket_type) { + if (socket_type == Network::Socket::Type::Datagram) { + auto udp_writer_config = config_.udp_writer_config(); + if (!Api::OsSysCallsSingleton::get().supportsUdpGso() || + udp_writer_config.typed_config().type_url().empty()) { + const std::string default_type_url = + "type.googleapis.com/envoy.config.listener.v3.UdpDefaultWriterOptions"; + udp_writer_config.mutable_typed_config()->set_type_url(default_type_url); + } + auto& config_factory = + Config::Utility::getAndCheckFactory( + udp_writer_config); + ProtobufTypes::MessagePtr message = Config::Utility::translateAnyToFactoryConfig( + udp_writer_config.typed_config(), validation_visitor_, config_factory); + udp_writer_factory_ = config_factory.createUdpPacketWriterFactory(*message); + } +} + void ListenerImpl::buildListenSocketOptions(Network::Socket::Type socket_type) { // The process-wide `signal()` handling may fail to handle SIGPIPE if overridden // in the process (i.e., on a mobile client). Some OSes support handling it at the socket layer: diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index aa1bd2ca1b0d..920f8a24e9b3 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -302,6 +302,9 @@ class ListenerImpl final : public Network::ListenerConfig, Network::ActiveUdpListenerFactory* udpListenerFactory() override { return udp_listener_factory_.get(); } + Network::UdpPacketWriterFactoryOptRef udpPacketWriterFactory() override { + return Network::UdpPacketWriterFactoryOptRef(std::ref(*udp_writer_factory_)); + } Network::ConnectionBalancer& connectionBalancer() override { return *connection_balancer_; } ResourceLimit& openConnections() override { return *open_connections_; } @@ -341,6 +344,7 @@ class ListenerImpl final : public Network::ListenerConfig, // Helpers for constructor. void buildAccessLog(); void buildUdpListenerFactory(Network::Socket::Type socket_type, uint32_t concurrency); + void buildUdpWriterFactory(Network::Socket::Type socket_type); void buildListenSocketOptions(Network::Socket::Type socket_type); void createListenerFilterFactories(Network::Socket::Type socket_type); void validateFilterChains(Network::Socket::Type socket_type); @@ -386,6 +390,7 @@ class ListenerImpl final : public Network::ListenerConfig, const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; Network::ActiveUdpListenerFactoryPtr udp_listener_factory_; + Network::UdpPacketWriterFactoryPtr udp_writer_factory_; Network::ConnectionBalancerPtr connection_balancer_; std::shared_ptr listener_factory_context_; FilterChainManagerImpl filter_chain_manager_; diff --git a/test/common/network/BUILD b/test/common/network/BUILD index cd05a87e8344..0557529c0a0a 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -196,23 +196,74 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "udp_listener_impl_test_base_lib", + hdrs = ["udp_listener_impl_test_base.h"], + deps = [ + "//source/common/event:dispatcher_lib", + "//source/common/network:address_lib", + "//source/common/network:listener_lib", + "//source/common/network:utility_lib", + "//source/common/stats:stats_lib", + "//test/mocks/network:network_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "udp_listener_impl_test", srcs = ["udp_listener_impl_test.cc"], tags = ["fails_on_windows"], deps = [ + ":udp_listener_impl_test_base_lib", + "//source/common/event:dispatcher_lib", + "//source/common/network:address_lib", + "//source/common/network:listener_lib", + "//source/common/network:socket_option_lib", + "//source/common/network:udp_packet_writer_handler_lib", + "//source/common/network:utility_lib", + "//source/common/stats:stats_lib", + "//test/common/network:listener_impl_test_base_lib", + "//test/mocks/network:network_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:environment_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:threadsafe_singleton_injector_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "udp_listener_impl_batch_writer_test", + srcs = ["udp_listener_impl_batch_writer_test.cc"], + tags = [ + # Skipping as quiche quic_gso_batch_writer.h does not exist on Windows + "skip_on_windows", + ], + deps = [ + ":udp_listener_impl_test_base_lib", "//source/common/event:dispatcher_lib", "//source/common/network:address_lib", "//source/common/network:listener_lib", "//source/common/network:socket_option_lib", + "//source/common/network:udp_packet_writer_handler_lib", "//source/common/network:utility_lib", "//source/common/stats:stats_lib", + "//source/extensions/quic_listeners/quiche:udp_gso_batch_writer_lib", "//test/common/network:listener_impl_test_base_lib", "//test/mocks/network:network_mocks", "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", + "@com_googlesource_quiche//:quic_test_tools_mock_syscall_wrapper_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/test/common/network/udp_listener_impl_batch_writer_test.cc b/test/common/network/udp_listener_impl_batch_writer_test.cc new file mode 100644 index 000000000000..959fd52515f1 --- /dev/null +++ b/test/common/network/udp_listener_impl_batch_writer_test.cc @@ -0,0 +1,279 @@ +#include +#include +#include +#include +#include + +#ifdef __GNUC__ +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wtype-limits" + +#include "quiche/quic/test_tools/quic_mock_syscall_wrapper.h" + +#pragma GCC diagnostic pop +#else +#include "quiche/quic/test_tools/quic_mock_syscall_wrapper.h" +#endif + +#include "envoy/config/core/v3/base.pb.h" + +#include "common/network/address_impl.h" +#include "common/network/socket_option_factory.h" +#include "common/network/socket_option_impl.h" +#include "common/network/udp_listener_impl.h" +#include "common/network/utility.h" + +#include "extensions/quic_listeners/quiche/udp_gso_batch_writer.h" + +#include "test/common/network/udp_listener_impl_test_base.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/server/mocks.h" +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/threadsafe_singleton_injector.h" +#include "test/test_common/utility.h" + +#include "absl/time/time.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::ReturnRef; + +namespace Envoy { +namespace Network { +namespace { + +size_t getPacketLength(const msghdr* msg) { + size_t length = 0; + for (size_t i = 0; i < msg->msg_iovlen; ++i) { + length += msg->msg_iov[i].iov_len; + } + return length; +} + +class UdpListenerImplBatchWriterTest : public UdpListenerImplTestBase { +public: + void SetUp() override { + // Set listening socket options and set UdpGsoBatchWriter + server_socket_->addOptions(SocketOptionFactory::buildIpPacketInfoOptions()); + server_socket_->addOptions(SocketOptionFactory::buildRxQueueOverFlowOptions()); + listener_ = std::make_unique( + dispatcherImpl(), server_socket_, listener_callbacks_, dispatcherImpl().timeSource()); + udp_packet_writer_ = std::make_unique( + server_socket_->ioHandle(), listener_config_.listenerScope()); + ON_CALL(listener_callbacks_, udpPacketWriter()).WillByDefault(ReturnRef(*udp_packet_writer_)); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, UdpListenerImplBatchWriterTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +/** + * Tests UDP Packet Writer To Send packets in Batches to a client + * 1. Setup a udp listener and client socket + * 2. Send different sized payloads to client. + * - Verify that the packets are buffered as long as payload + * length matches gso_size. + * - When payload size > gso_size verify that the new payload is + * buffered and already buffered packets are sent to client + * - When payload size < gso_size verify that the new payload is + * sent along with the already buffered payloads. + * 3. Call UdpPacketWriter's External Flush + * - Verify that the internal buffer is emptied and the + * total_bytes_sent counter is updated accordingly. + */ +TEST_P(UdpListenerImplBatchWriterTest, SendData) { + EXPECT_TRUE(udp_packet_writer_->isBatchMode()); + Address::InstanceConstSharedPtr send_from_addr = getNonDefaultSourceAddress(); + + absl::FixedArray payloads{"length7", "length7", "len<7", + "length7", "length7", "length>7"}; + std::string internal_buffer(""); + std::string last_buffered(""); + std::list pkts_to_send; + bool send_buffered_pkts = false; + + // Get initial value of total_bytes_sent + uint64_t total_bytes_sent = + listener_config_.listenerScope().counterFromString("total_bytes_sent").value(); + + for (const auto& payload : payloads) { + Buffer::InstancePtr buffer(new Buffer::OwnedImpl()); + buffer->add(payload); + UdpSendData send_data{send_from_addr->ip(), *client_.localAddress(), *buffer}; + + auto send_result = listener_->send(send_data); + EXPECT_TRUE(send_result.ok()) << "send() failed : " << send_result.err_->getErrorDetails(); + EXPECT_EQ(send_result.rc_, payload.length()); + + // Verify udp_packet_writer stats for batch writing + if (internal_buffer.length() == 0 || /* internal buffer is empty*/ + payload.compare(last_buffered) == 0) { /*len(payload) == gso_size*/ + pkts_to_send.emplace_back(payload); + internal_buffer.append(payload); + last_buffered = payload; + } else if (payload.compare(last_buffered) < 0) { /*len(payload) < gso_size*/ + pkts_to_send.emplace_back(payload); + internal_buffer.clear(); + last_buffered.clear(); + send_buffered_pkts = true; + } else { /*len(payload) > gso_size*/ + internal_buffer = payload; + last_buffered = payload; + send_buffered_pkts = true; + } + + EXPECT_EQ(listener_config_.listenerScope() + .gaugeFromString("internal_buffer_size", Stats::Gauge::ImportMode::NeverImport) + .value(), + internal_buffer.length()); + + // Verify that the total_bytes_sent is only updated when the packets + // are actually sent to the client, and not on being buffered. + if (send_buffered_pkts) { + for (const auto& pkt : pkts_to_send) { + total_bytes_sent += pkt.length(); + } + pkts_to_send.clear(); + if (last_buffered.length() != 0) { + pkts_to_send.emplace_back(last_buffered); + } + send_buffered_pkts = false; + } + EXPECT_EQ(listener_config_.listenerScope().counterFromString("total_bytes_sent").value(), + total_bytes_sent); + } + + // Test External Flush + auto flush_result = udp_packet_writer_->flush(); + EXPECT_TRUE(flush_result.ok()); + EXPECT_EQ(listener_config_.listenerScope() + .gaugeFromString("internal_buffer_size", Stats::Gauge::ImportMode::NeverImport) + .value(), + 0); + total_bytes_sent += payloads.back().length(); + + EXPECT_EQ(listener_config_.listenerScope().counterFromString("total_bytes_sent").value(), + total_bytes_sent); +} + +/** + * Tests UDP Packet writer behavior when socket is write-blocked. + * 1. Setup the udp_listener and have a payload buffered in the internal buffer. + * 2. Then set the socket to return EWOULDBLOCK error on sendmsg and write a + * different sized buffer to the packet writer. + * - Ensure that a buffer shorter than the initial buffer is added to the + * Internal Buffer. + * - A buffer longer than the initial buffer should not get appended to the + * Internal Buffer. + */ +TEST_P(UdpListenerImplBatchWriterTest, WriteBlocked) { + // Quic Mock Objects + quic::test::MockQuicSyscallWrapper os_sys_calls; + quic::ScopedGlobalSyscallWrapperOverride os_calls(&os_sys_calls); + + // The initial payload to be buffered + std::string initial_payload("length7"); + + // Get initial value of total_bytes_sent + uint64_t total_bytes_sent = + listener_config_.listenerScope().counterFromString("total_bytes_sent").value(); + + // Possible following payloads to be sent after the initial payload + absl::FixedArray following_payloads{"length<7", "len<7"}; + + for (const auto& following_payload : following_payloads) { + std::string internal_buffer(""); + + // First have initial payload added to the udp_packet_writer's internal buffer. + Buffer::InstancePtr initial_buffer(new Buffer::OwnedImpl()); + initial_buffer->add(initial_payload); + UdpSendData initial_send_data{send_to_addr_->ip(), *server_socket_->localAddress(), + *initial_buffer}; + auto send_result = listener_->send(initial_send_data); + internal_buffer.append(initial_payload); + EXPECT_TRUE(send_result.ok()); + EXPECT_EQ(send_result.rc_, initial_payload.length()); + EXPECT_FALSE(udp_packet_writer_->isWriteBlocked()); + EXPECT_EQ(listener_config_.listenerScope() + .gaugeFromString("internal_buffer_size", Stats::Gauge::ImportMode::NeverImport) + .value(), + initial_payload.length()); + EXPECT_EQ(listener_config_.listenerScope().counterFromString("total_bytes_sent").value(), + total_bytes_sent); + + // Mock the socket to be write blocked on sendmsg syscall + EXPECT_CALL(os_sys_calls, Sendmsg(_, _, _)) + .WillOnce(Invoke([](int /*sockfd*/, const msghdr* /*msg*/, int /*flags*/) { + errno = EWOULDBLOCK; + return -1; + })); + + // Now send the following payload + Buffer::InstancePtr following_buffer(new Buffer::OwnedImpl()); + following_buffer->add(following_payload); + UdpSendData following_send_data{send_to_addr_->ip(), *server_socket_->localAddress(), + *following_buffer}; + send_result = listener_->send(following_send_data); + + if (following_payload.length() < initial_payload.length()) { + // The following payload should get buffered if it is + // shorter than initial payload + EXPECT_TRUE(send_result.ok()); + EXPECT_EQ(send_result.rc_, following_payload.length()); + EXPECT_FALSE(udp_packet_writer_->isWriteBlocked()); + internal_buffer.append(following_payload); + // Send another packet and verify that writer gets blocked later + EXPECT_CALL(os_sys_calls, Sendmsg(_, _, _)) + .WillOnce(Invoke([](int /*sockfd*/, const msghdr* /*msg*/, int /*flags*/) { + errno = EWOULDBLOCK; + return -1; + })); + following_buffer->add(following_payload); + UdpSendData final_send_data{send_to_addr_->ip(), *server_socket_->localAddress(), + *following_buffer}; + send_result = listener_->send(final_send_data); + } + + EXPECT_FALSE(send_result.ok()); + EXPECT_EQ(send_result.rc_, 0); + EXPECT_TRUE(udp_packet_writer_->isWriteBlocked()); + EXPECT_EQ(listener_config_.listenerScope().counterFromString("total_bytes_sent").value(), + total_bytes_sent); + EXPECT_EQ(listener_config_.listenerScope() + .gaugeFromString("internal_buffer_size", Stats::Gauge::ImportMode::NeverImport) + .value(), + internal_buffer.length()); + + // Reset write blocked status and verify correct buffer is flushed + udp_packet_writer_->setWritable(); + EXPECT_CALL(os_sys_calls, Sendmsg(_, _, _)) + .WillOnce(Invoke([&](int /*sockfd*/, const msghdr* msg, int /*flags*/) { + EXPECT_EQ(internal_buffer.length(), getPacketLength(msg)); + return internal_buffer.length(); + })); + auto flush_result = udp_packet_writer_->flush(); + EXPECT_TRUE(flush_result.ok()); + EXPECT_EQ(flush_result.rc_, 0); + EXPECT_FALSE(udp_packet_writer_->isWriteBlocked()); + EXPECT_EQ(listener_config_.listenerScope() + .gaugeFromString("internal_buffer_size", Stats::Gauge::ImportMode::NeverImport) + .value(), + 0); + total_bytes_sent += internal_buffer.length(); + EXPECT_EQ(listener_config_.listenerScope().counterFromString("total_bytes_sent").value(), + total_bytes_sent); + } +} + +} // namespace +} // namespace Network +} // namespace Envoy diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index 11b4e5346ead..c2d1a4216bb6 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -11,9 +11,10 @@ #include "common/network/socket_option_factory.h" #include "common/network/socket_option_impl.h" #include "common/network/udp_listener_impl.h" +#include "common/network/udp_packet_writer_handler_impl.h" #include "common/network/utility.h" -#include "test/common/network/listener_impl_test_base.h" +#include "test/common/network/udp_listener_impl_test_base.h" #include "test/mocks/api/mocks.h" #include "test/mocks/network/mocks.h" #include "test/test_common/environment.h" @@ -28,6 +29,7 @@ using testing::_; using testing::Invoke; using testing::Return; +using testing::ReturnRef; namespace Envoy { namespace Network { @@ -45,76 +47,23 @@ class MockSupportsUdpGro : public Api::OsSysCallsImpl { MOCK_METHOD(bool, supportsUdpGro, (), (const)); }; -class UdpListenerImplTest : public ListenerImplTestBase { +class UdpListenerImplTest : public UdpListenerImplTestBase { public: - UdpListenerImplTest() - : server_socket_(createServerSocket(true)), send_to_addr_(getServerLoopbackAddress()) { - time_system_.advanceTimeWait(std::chrono::milliseconds(100)); + void SetUp() override { ON_CALL(udp_gro_syscall_, supportsUdpGro()).WillByDefault(Return(false)); - } - void SetUp() override { // Set listening socket options. server_socket_->addOptions(SocketOptionFactory::buildIpPacketInfoOptions()); server_socket_->addOptions(SocketOptionFactory::buildRxQueueOverFlowOptions()); if (Api::OsSysCallsSingleton::get().supportsUdpGro()) { server_socket_->addOptions(SocketOptionFactory::buildUdpGroOptions()); } - listener_ = std::make_unique( dispatcherImpl(), server_socket_, listener_callbacks_, dispatcherImpl().timeSource()); + udp_packet_writer_ = std::make_unique(server_socket_->ioHandle()); + ON_CALL(listener_callbacks_, udpPacketWriter()).WillByDefault(ReturnRef(*udp_packet_writer_)); } -protected: - Address::Instance* getServerLoopbackAddress() { - if (version_ == Address::IpVersion::v4) { - return new Address::Ipv4Instance(Network::Test::getLoopbackAddressString(version_), - server_socket_->localAddress()->ip()->port()); - } - return new Address::Ipv6Instance(Network::Test::getLoopbackAddressString(version_), - server_socket_->localAddress()->ip()->port()); - } - - SocketSharedPtr createServerSocket(bool bind) { - // Set IP_FREEBIND to allow sendmsg to send with non-local IPv6 source address. - return std::make_shared(Network::Test::getAnyAddress(version_), -#ifdef IP_FREEBIND - SocketOptionFactory::buildIpFreebindOptions(), -#else - nullptr, -#endif - bind); - } - - // Validates receive data, source/destination address and received time. - void validateRecvCallbackParams(const UdpRecvData& data, size_t num_packet_per_recv) { - ASSERT_NE(data.addresses_.local_, nullptr); - - ASSERT_NE(data.addresses_.peer_, nullptr); - ASSERT_NE(data.addresses_.peer_->ip(), nullptr); - - EXPECT_EQ(data.addresses_.local_->asString(), send_to_addr_->asString()); - - EXPECT_EQ(data.addresses_.peer_->ip()->addressAsString(), - client_.localAddress()->ip()->addressAsString()); - - EXPECT_EQ(*data.addresses_.local_, *send_to_addr_); - - EXPECT_EQ(time_system_.monotonicTime(), - data.receive_time_ + - std::chrono::milliseconds( - (num_packets_received_by_listener_ % num_packet_per_recv) * 100)); - // Advance time so that next onData() should have different received time. - time_system_.advanceTimeWait(std::chrono::milliseconds(100)); - ++num_packets_received_by_listener_; - } - - SocketSharedPtr server_socket_; - Network::Test::UdpSyncPeer client_{GetParam()}; - Address::InstanceConstSharedPtr send_to_addr_; - MockUdpListenerCallbacks listener_callbacks_; - std::unique_ptr listener_; - size_t num_packets_received_by_listener_{0}; NiceMock udp_gro_syscall_; TestThreadsafeSingletonInjector os_calls{&udp_gro_syscall_}; }; @@ -342,35 +291,12 @@ TEST_P(UdpListenerImplTest, UdpListenerRecvMsgError) { * address. */ TEST_P(UdpListenerImplTest, SendData) { + EXPECT_FALSE(udp_packet_writer_->isBatchMode()); const std::string payload("hello world"); Buffer::InstancePtr buffer(new Buffer::OwnedImpl()); buffer->add(payload); - // Use a self address that is unlikely to be picked by source address discovery - // algorithm if not specified in recvmsg/recvmmsg. Port is not taken into - // consideration. - Address::InstanceConstSharedPtr send_from_addr; - if (version_ == Address::IpVersion::v4) { - // Linux kernel regards any 127.x.x.x as local address. But Mac OS doesn't. - send_from_addr = std::make_shared( -#ifndef __APPLE__ - "127.1.2.3", -#else - "127.0.0.1", -#endif - server_socket_->localAddress()->ip()->port()); - } else { - // Only use non-local v6 address if IP_FREEBIND is supported. Otherwise use - // ::1 to avoid EINVAL error. Unfortunately this can't verify that sendmsg with - // customized source address is doing the work because kernel also picks ::1 - // if it's not specified in cmsghdr. - send_from_addr = std::make_shared( -#ifdef IP_FREEBIND - "::9", -#else - "::1", -#endif - server_socket_->localAddress()->ip()->port()); - } + + Address::InstanceConstSharedPtr send_from_addr = getNonDefaultSourceAddress(); UdpSendData send_data{send_from_addr->ip(), *client_.localAddress(), *buffer}; @@ -384,6 +310,11 @@ TEST_P(UdpListenerImplTest, SendData) { EXPECT_EQ(bytes_to_read, data.buffer_->length()); EXPECT_EQ(send_from_addr->asString(), data.addresses_.peer_->asString()); EXPECT_EQ(data.buffer_->toString(), payload); + + // Verify External Flush is a No-op + auto flush_result = udp_packet_writer_->flush(); + EXPECT_TRUE(flush_result.ok()); + EXPECT_EQ(0, flush_result.rc_); } /** @@ -399,10 +330,25 @@ TEST_P(UdpListenerImplTest, SendDataError) { // Inject mocked OsSysCalls implementation to mock a write failure. Api::MockOsSysCalls os_sys_calls; TestThreadsafeSingletonInjector os_calls(&os_sys_calls); + EXPECT_CALL(os_sys_calls, sendmsg(_, _, _)) - .WillOnce(Return(Api::SysCallSizeResult{-1, SOCKET_ERROR_NOT_SUP})); + .WillOnce(Return(Api::SysCallSizeResult{-1, SOCKET_ERROR_AGAIN})); auto send_result = listener_->send(send_data); EXPECT_FALSE(send_result.ok()); + EXPECT_EQ(send_result.err_->getErrorCode(), Api::IoError::IoErrorCode::Again); + // Failed write shouldn't drain the data. + EXPECT_EQ(payload.length(), buffer->length()); + // Verify the writer is set to blocked + EXPECT_TRUE(udp_packet_writer_->isWriteBlocked()); + + // Reset write_blocked status + udp_packet_writer_->setWritable(); + EXPECT_FALSE(udp_packet_writer_->isWriteBlocked()); + + EXPECT_CALL(os_sys_calls, sendmsg(_, _, _)) + .WillOnce(Return(Api::SysCallSizeResult{-1, SOCKET_ERROR_NOT_SUP})); + send_result = listener_->send(send_data); + EXPECT_FALSE(send_result.ok()); EXPECT_EQ(send_result.err_->getErrorCode(), Api::IoError::IoErrorCode::NoSupport); // Failed write shouldn't drain the data. EXPECT_EQ(payload.length(), buffer->length()); diff --git a/test/common/network/udp_listener_impl_test_base.h b/test/common/network/udp_listener_impl_test_base.h new file mode 100644 index 000000000000..2547986a316a --- /dev/null +++ b/test/common/network/udp_listener_impl_test_base.h @@ -0,0 +1,123 @@ +#include +#include +#include +#include + +#include "envoy/config/core/v3/base.pb.h" + +#include "common/network/address_impl.h" +#include "common/network/socket_option_factory.h" +#include "common/network/socket_option_impl.h" +#include "common/network/udp_listener_impl.h" +#include "common/network/udp_packet_writer_handler_impl.h" +#include "common/network/utility.h" + +#include "test/common/network/listener_impl_test_base.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/server/mocks.h" +#include "test/test_common/environment.h" +#include "test/test_common/network_utility.h" +#include "test/test_common/threadsafe_singleton_injector.h" +#include "test/test_common/utility.h" + +#include "absl/time/time.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Network { + +class UdpListenerImplTestBase : public ListenerImplTestBase { +public: + UdpListenerImplTestBase() + : server_socket_(createServerSocket(true)), send_to_addr_(getServerLoopbackAddress()) { + time_system_.advanceTimeWait(std::chrono::milliseconds(100)); + } + +protected: + Address::Instance* getServerLoopbackAddress() { + if (version_ == Address::IpVersion::v4) { + return new Address::Ipv4Instance(Network::Test::getLoopbackAddressString(version_), + server_socket_->localAddress()->ip()->port()); + } + return new Address::Ipv6Instance(Network::Test::getLoopbackAddressString(version_), + server_socket_->localAddress()->ip()->port()); + } + + SocketSharedPtr createServerSocket(bool bind) { + // Set IP_FREEBIND to allow sendmsg to send with non-local IPv6 source address. + return std::make_shared(Network::Test::getAnyAddress(version_), +#ifdef IP_FREEBIND + SocketOptionFactory::buildIpFreebindOptions(), +#else + nullptr, +#endif + bind); + } + + Address::InstanceConstSharedPtr getNonDefaultSourceAddress() { + // Use a self address that is unlikely to be picked by source address discovery + // algorithm if not specified in recvmsg/recvmmsg. Port is not taken into + // consideration. + Address::InstanceConstSharedPtr send_from_addr; + if (version_ == Address::IpVersion::v4) { + // Linux kernel regards any 127.x.x.x as local address. But Mac OS doesn't. + send_from_addr = std::make_shared( +#ifndef __APPLE__ + "127.1.2.3", +#else + "127.0.0.1", +#endif + server_socket_->localAddress()->ip()->port()); + } else { + // Only use non-local v6 address if IP_FREEBIND is supported. Otherwise use + // ::1 to avoid EINVAL error. Unfortunately this can't verify that sendmsg with + // customized source address is doing the work because kernel also picks ::1 + // if it's not specified in cmsghdr. + send_from_addr = std::make_shared( +#ifdef IP_FREEBIND + "::9", +#else + "::1", +#endif + server_socket_->localAddress()->ip()->port()); + } + return send_from_addr; + } + + // Validates receive data, source/destination address and received time. + void validateRecvCallbackParams(const UdpRecvData& data, size_t num_packet_per_recv) { + ASSERT_NE(data.addresses_.local_, nullptr); + + ASSERT_NE(data.addresses_.peer_, nullptr); + ASSERT_NE(data.addresses_.peer_->ip(), nullptr); + + EXPECT_EQ(data.addresses_.local_->asString(), send_to_addr_->asString()); + + EXPECT_EQ(data.addresses_.peer_->ip()->addressAsString(), + client_.localAddress()->ip()->addressAsString()); + + EXPECT_EQ(*data.addresses_.local_, *send_to_addr_); + + EXPECT_EQ(time_system_.monotonicTime(), + data.receive_time_ + + std::chrono::milliseconds( + (num_packets_received_by_listener_ % num_packet_per_recv) * 100)); + // Advance time so that next onData() should have different received time. + time_system_.advanceTimeWait(std::chrono::milliseconds(100)); + ++num_packets_received_by_listener_; + } + + SocketSharedPtr server_socket_; + Network::Test::UdpSyncPeer client_{GetParam()}; + Address::InstanceConstSharedPtr send_to_addr_; + NiceMock listener_callbacks_; + NiceMock listener_config_; + std::unique_ptr listener_; + size_t num_packets_received_by_listener_{0}; + Network::UdpPacketWriterPtr udp_packet_writer_; +}; + +} // namespace Network +} // namespace Envoy diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index 8654352291de..7c1bd0d80ae1 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -70,6 +70,7 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam(&new_session.file_event_cb_), Return(nullptr))); + // Internal Buffer is Empty, flush will be a no-op + ON_CALL(callbacks_.udp_listener_, flush()) + .WillByDefault( + InvokeWithoutArgs([]() -> Api::IoCallUint64Result { return makeNoError(0); })); } void checkTransferStats(uint64_t rx_bytes, uint64_t rx_datagrams, uint64_t tx_bytes, diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index 29ae0a89eb28..bb259455a7c1 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -34,6 +34,7 @@ envoy_cc_test( ], deps = [ "//source/common/network:io_socket_error_lib", + "//source/common/network:udp_packet_writer_handler_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_packet_writer_lib", "//test/mocks/api:api_mocks", "//test/mocks/network:network_mocks", @@ -192,6 +193,7 @@ envoy_cc_test( "//source/extensions/quic_listeners/quiche:active_quic_listener_config_lib", "//source/extensions/quic_listeners/quiche:active_quic_listener_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_utils_lib", + "//source/extensions/quic_listeners/quiche:udp_gso_batch_writer_lib", "//source/server:configuration_lib", "//test/mocks/network:network_mocks", "//test/mocks/server:instance_mocks", diff --git a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc index 747452ccdf78..b41b6bdd311d 100644 --- a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc +++ b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc @@ -43,6 +43,7 @@ #include "extensions/quic_listeners/quiche/active_quic_listener_config.h" #include "extensions/quic_listeners/quiche/platform/envoy_quic_clock.h" #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/udp_gso_batch_writer.h" using testing::Return; using testing::ReturnRef; @@ -111,6 +112,18 @@ class ActiveQuicListenerTest : public QuicMultiVersionTest { ON_CALL(listener_config_, listenSocketFactory()).WillByDefault(ReturnRef(socket_factory_)); ON_CALL(socket_factory_, getListenSocket()).WillByDefault(Return(listen_socket_)); + // Use UdpGsoBatchWriter to perform non-batched writes for the purpose of this test + ON_CALL(listener_config_, udpPacketWriterFactory()) + .WillByDefault(Return( + std::reference_wrapper(udp_packet_writer_factory_))); + ON_CALL(udp_packet_writer_factory_, createUdpPacketWriter(_, _)) + .WillByDefault(Invoke( + [&](Network::IoHandle& io_handle, Stats::Scope& scope) -> Network::UdpPacketWriterPtr { + Network::UdpPacketWriterPtr udp_packet_writer = + std::make_unique(io_handle, scope); + return udp_packet_writer; + })); + listener_factory_ = createQuicListenerFactory(yamlForQuicConfig()); EXPECT_CALL(listener_config_, filterChainManager()).WillOnce(ReturnRef(filter_chain_manager_)); quic_listener_ = @@ -179,8 +192,8 @@ class ActiveQuicListenerTest : public QuicMultiVersionTest { client_sockets_.push_back(std::make_unique(local_address_, nullptr, /*bind*/ false)); Buffer::OwnedImpl payload = generateChloPacketToSend( quic_version_, quic_config_, ActiveQuicListenerPeer::cryptoConfig(*quic_listener_), - connection_id, clock_, envoyAddressInstanceToQuicSocketAddress(local_address_), - envoyAddressInstanceToQuicSocketAddress(local_address_), "test.example.org"); + connection_id, clock_, envoyIpAddressToQuicSocketAddress(local_address_->ip()), + envoyIpAddressToQuicSocketAddress(local_address_->ip()), "test.example.org"); Buffer::RawSliceVector slice = payload.getRawSlices(); ASSERT_EQ(1u, slice.size()); // Send a full CHLO to finish 0-RTT handshake. @@ -243,6 +256,7 @@ class ActiveQuicListenerTest : public QuicMultiVersionTest { std::shared_ptr read_filter_; Network::MockConnectionCallbacks network_connection_callbacks_; NiceMock listener_config_; + NiceMock udp_packet_writer_factory_; quic::QuicConfig quic_config_; Server::ConnectionHandlerImpl connection_handler_; std::unique_ptr quic_listener_; diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc index fb15815fa1db..c5b6e6c2e7af 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc @@ -128,7 +128,7 @@ class EnvoyQuicDispatcherTest : public QuicMultiVersionTest, EnvoyQuicClock clock(*dispatcher_); Buffer::OwnedImpl payload = generateChloPacketToSend( quic_version_, quic_config_, crypto_config_, connection_id_, clock, - envoyAddressInstanceToQuicSocketAddress(listen_socket_->localAddress()), peer_addr, + envoyIpAddressToQuicSocketAddress(listen_socket_->localAddress()->ip()), peer_addr, "test.example.org"); Buffer::RawSliceVector slice = payload.getRawSlices(); ASSERT(slice.size() == 1); @@ -139,7 +139,7 @@ class EnvoyQuicDispatcherTest : public QuicMultiVersionTest, quic::test::ConstructReceivedPacket(*encrypted_packet, clock.Now())); envoy_quic_dispatcher_.ProcessPacket( - envoyAddressInstanceToQuicSocketAddress(listen_socket_->localAddress()), peer_addr, + envoyIpAddressToQuicSocketAddress(listen_socket_->localAddress()->ip()), peer_addr, *received_packet); if (should_buffer) { @@ -165,7 +165,7 @@ class EnvoyQuicDispatcherTest : public QuicMultiVersionTest, auto envoy_connection = static_cast(session); EXPECT_EQ("test.example.org", envoy_connection->requestedServerName()); EXPECT_EQ(peer_addr, - envoyAddressInstanceToQuicSocketAddress(envoy_connection->remoteAddress())); + envoyIpAddressToQuicSocketAddress(envoy_connection->remoteAddress()->ip())); ASSERT(envoy_connection->localAddress() != nullptr); EXPECT_EQ(*listen_socket_->localAddress(), *envoy_connection->localAddress()); } diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc index cd5004c39c2d..68d606ea54b4 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc @@ -39,7 +39,7 @@ TEST(EnvoyQuicUtilsTest, ConversionBetweenQuicAddressAndEnvoyAddress) { Network::Address::InstanceConstSharedPtr envoy_addr = quicAddressToEnvoyAddressInstance(quic_addr); EXPECT_EQ(quic_addr.ToString(), envoy_addr->asStringView()); - EXPECT_EQ(quic_addr, envoyAddressInstanceToQuicSocketAddress(envoy_addr)); + EXPECT_EQ(quic_addr, envoyIpAddressToQuicSocketAddress(envoy_addr->ip())); } } diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc index 0c6232fb8e50..cb22532e69bb 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc @@ -5,6 +5,7 @@ #include "common/network/address_impl.h" #include "common/network/io_socket_error_impl.h" +#include "common/network/udp_packet_writer_handler_impl.h" #include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" @@ -22,7 +23,8 @@ namespace Quic { class EnvoyQuicWriterTest : public ::testing::Test { public: - EnvoyQuicWriterTest() : envoy_quic_writer_(socket_) { + EnvoyQuicWriterTest() + : envoy_quic_writer_(std::make_unique(socket_.ioHandle())) { self_address_.FromString("::"); quic::QuicIpAddress peer_ip; peer_ip.FromString("::1"); diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index b735399ade3d..46287d7a5191 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -32,6 +32,7 @@ #include "common/network/connection_balancer_impl.h" #include "common/network/filter_impl.h" #include "common/network/listen_socket_impl.h" +#include "common/network/udp_default_writer_config.h" #include "common/stats/isolated_store_impl.h" #include "server/active_raw_udp_listener_config.h" @@ -698,7 +699,8 @@ class FakeUpstream : Logger::Loggable, public: FakeListener(FakeUpstream& parent) : parent_(parent), name_("fake_upstream"), - udp_listener_factory_(std::make_unique()) {} + udp_listener_factory_(std::make_unique()), + udp_writer_factory_(std::make_unique()) {} private: // Network::ListenerConfig @@ -718,6 +720,9 @@ class FakeUpstream : Logger::Loggable, Network::ActiveUdpListenerFactory* udpListenerFactory() override { return udp_listener_factory_.get(); } + Network::UdpPacketWriterFactoryOptRef udpPacketWriterFactory() override { + return Network::UdpPacketWriterFactoryOptRef(std::ref(*udp_writer_factory_)); + } Network::ConnectionBalancer& connectionBalancer() override { return connection_balancer_; } envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; @@ -736,6 +741,7 @@ class FakeUpstream : Logger::Loggable, const std::string name_; Network::NopConnectionBalancerImpl connection_balancer_; const Network::ActiveUdpListenerFactoryPtr udp_listener_factory_; + const Network::UdpPacketWriterFactoryPtr udp_writer_factory_; BasicResourceLimitImpl connection_resource_; const std::vector empty_access_logs_; }; diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index be6d86765d0d..596afe6ffbea 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -142,6 +142,7 @@ class MockUdpListenerCallbacks : public UdpListenerCallbacks { MOCK_METHOD(void, onReadReady, ()); MOCK_METHOD(void, onWriteReady, (const Socket& socket)); MOCK_METHOD(void, onReceiveError, (Api::IoError::IoErrorCode err)); + MOCK_METHOD(Network::UdpPacketWriter&, udpPacketWriter, ()); }; class MockDrainDecision : public DrainDecision { @@ -328,6 +329,14 @@ class MockListenSocketFactory : public ListenSocketFactory { MOCK_METHOD(SocketOptRef, sharedSocket, (), (const)); }; +class MockUdpPacketWriterFactory : public UdpPacketWriterFactory { +public: + MockUdpPacketWriterFactory() = default; + + MOCK_METHOD(Network::UdpPacketWriterPtr, createUdpPacketWriter, + (Network::IoHandle&, Stats::Scope&), ()); +}; + class MockListenerConfig : public ListenerConfig { public: MockListenerConfig(); @@ -345,6 +354,7 @@ class MockListenerConfig : public ListenerConfig { MOCK_METHOD(uint64_t, listenerTag, (), (const)); MOCK_METHOD(const std::string&, name, (), (const)); MOCK_METHOD(Network::ActiveUdpListenerFactory*, udpListenerFactory, ()); + MOCK_METHOD(Network::UdpPacketWriterFactoryOptRef, udpPacketWriterFactory, ()); MOCK_METHOD(ConnectionBalancer&, connectionBalancer, ()); MOCK_METHOD(ResourceLimit&, openConnections, ()); @@ -455,6 +465,22 @@ class MockTransportSocketCallbacks : public TransportSocketCallbacks { testing::NiceMock connection_; }; +class MockUdpPacketWriter : public UdpPacketWriter { +public: + MockUdpPacketWriter() = default; + + MOCK_METHOD(Api::IoCallUint64Result, writePacket, + (const Buffer::Instance& buffer, const Address::Ip* local_ip, + const Address::Instance& peer_address)); + MOCK_METHOD(bool, isWriteBlocked, (), (const)); + MOCK_METHOD(void, setWritable, ()); + MOCK_METHOD(uint64_t, getMaxPacketSize, (const Address::Instance& peer_address), (const)); + MOCK_METHOD(bool, isBatchMode, (), (const)); + MOCK_METHOD(Network::UdpPacketWriterBuffer, getNextWriteLocation, + (const Address::Ip* local_ip, const Address::Instance& peer_address)); + MOCK_METHOD(Api::IoCallUint64Result, flush, ()); +}; + class MockUdpListener : public UdpListener { public: MockUdpListener(); @@ -466,6 +492,7 @@ class MockUdpListener : public UdpListener { MOCK_METHOD(Event::Dispatcher&, dispatcher, ()); MOCK_METHOD(Address::InstanceConstSharedPtr&, localAddress, (), (const)); MOCK_METHOD(Api::IoCallUint64Result, send, (const UdpSendData&)); + MOCK_METHOD(Api::IoCallUint64Result, flush, ()); Event::MockDispatcher dispatcher_; }; diff --git a/test/server/BUILD b/test/server/BUILD index 007ae24ab5d0..84a3d3fb8899 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -75,6 +75,7 @@ envoy_cc_test( "//source/common/config:utility_lib", "//source/common/network:address_lib", "//source/common/network:connection_balancer_lib", + "//source/common/network:udp_default_writer_config", "//source/common/stats:stats_lib", "//source/server:active_raw_udp_listener_config", "//source/server:connection_handler_lib", @@ -274,6 +275,7 @@ envoy_cc_test( ":utility_lib", "//source/extensions/quic_listeners/quiche:active_quic_listener_config_lib", "//source/extensions/quic_listeners/quiche:quic_transport_socket_factory_lib", + "//source/extensions/quic_listeners/quiche:udp_gso_batch_writer_config_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//test/test_common:threadsafe_singleton_injector_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 7fcea249bbd4..ebcc66a8c656 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -11,6 +11,7 @@ #include "common/network/connection_balancer_impl.h" #include "common/network/io_socket_handle_impl.h" #include "common/network/raw_buffer_socket.h" +#include "common/network/udp_default_writer_config.h" #include "common/network/utility.h" #include "server/connection_handler_impl.h" @@ -56,7 +57,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> filter_chain_manager = nullptr) - : parent_(parent), socket_(std::make_shared()), + : parent_(parent), socket_(std::make_shared>()), socket_factory_(std::move(socket_factory)), tag_(tag), bind_to_port_(bind_to_port), hand_off_restored_destination_connections_(hand_off_restored_destination_connections), name_(name), listener_filters_timeout_(listener_filters_timeout), @@ -69,6 +70,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable(listener_name) .createActiveUdpListenerFactory(dummy, /*concurrency=*/1); + udp_writer_factory_ = std::make_unique(); ON_CALL(*socket_, socketType()).WillByDefault(Return(socket_type)); } @@ -96,6 +98,9 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable socket_; + std::shared_ptr> socket_; Network::ListenSocketFactorySharedPtr socket_factory_; uint64_t tag_; bool bind_to_port_; @@ -120,6 +125,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable udp_listener_factory_; + std::unique_ptr udp_writer_factory_; Network::ConnectionBalancerPtr connection_balancer_; BasicResourceLimitImpl open_connections_; const std::vector empty_access_logs_; @@ -155,6 +161,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable udp_gso_syscall_; + TestThreadsafeSingletonInjector os_calls{&udp_gso_syscall_}; +}; TEST_F(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryAndSslContext) { const std::string yaml = TestEnvironment::substitute(R"EOF( @@ -44,10 +53,15 @@ TEST_F(ListenerManagerImplQuicOnlyTest, QuicListenerFactoryAndSslContext) { reuse_port: true udp_listener_config: udp_listener_name: "quiche_quic_listener" +udp_writer_config: + name: "udp_gso_batch_writer" + typed_config: + "@type": type.googleapis.com/envoy.config.listener.v3.UdpGsoBatchWriterOptions )EOF", Network::Address::IpVersion::v4); envoy::config::listener::v3::Listener listener_proto = parseListenerFromV3Yaml(yaml); + ON_CALL(udp_gso_syscall_, supportsUdpGso()).WillByDefault(Return(true)); EXPECT_CALL(server_.random_, uuid()); expectCreateListenSocket(envoy::config::core::v3::SocketOption::STATE_PREBIND, #ifdef SO_RXQ_OVFL // SO_REUSEPORT is on as configured @@ -73,19 +87,23 @@ reuse_port: true /* expected_sockopt_name */ SO_REUSEPORT, /* expected_value */ 1, /* expected_num_calls */ 1); -#ifdef UDP_GRO if (Api::OsSysCallsSingleton::get().supportsUdpGro()) { expectSetsockopt(/* expected_sockopt_level */ SOL_UDP, /* expected_sockopt_name */ UDP_GRO, /* expected_value */ 1, /* expected_num_calls */ 1); } -#endif manager_->addOrUpdateListener(listener_proto, "", true); EXPECT_EQ(1u, manager_->listeners().size()); EXPECT_FALSE(manager_->listeners()[0].get().udpListenerFactory()->isTransportConnectionless()); - manager_->listeners().front().get().listenSocketFactory().getListenSocket(); + Network::SocketSharedPtr listen_socket = + manager_->listeners().front().get().listenSocketFactory().getListenSocket(); + + Network::UdpPacketWriterPtr udp_packet_writer = + manager_->listeners().front().get().udpPacketWriterFactory()->get().createUdpPacketWriter( + listen_socket->ioHandle(), manager_->listeners()[0].get().listenerScope()); + EXPECT_TRUE(udp_packet_writer->isBatchMode()); // No filter chain found with non-matching transport protocol. EXPECT_EQ(nullptr, findFilterChain(1234, "127.0.0.1", "", "tls", {}, "8.8.8.8", 111)); diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 85c5c14e76a8..c47c22f44b6e 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -4699,6 +4699,28 @@ name: foo EXPECT_EQ(0UL, manager_->listeners().size()); } +// This test verifies that on default initialization the UDP Packet Writer +// is initialized in passthrough mode. (i.e. by using UdpDefaultWriter). +TEST_F(ListenerManagerImplTest, UdpDefaultWriterConfig) { + const envoy::config::listener::v3::Listener listener = parseListenerFromV3Yaml(R"EOF( +address: + socket_address: + address: 127.0.0.1 + protocol: UDP + port_value: 1234 +filter_chains: + filters: [] + )EOF"); + manager_->addOrUpdateListener(listener, "", true); + EXPECT_EQ(1U, manager_->listeners().size()); + Network::SocketSharedPtr listen_socket = + manager_->listeners().front().get().listenSocketFactory().getListenSocket(); + Network::UdpPacketWriterPtr udp_packet_writer = + manager_->listeners().front().get().udpPacketWriterFactory()->get().createUdpPacketWriter( + listen_socket->ioHandle(), manager_->listeners()[0].get().listenerScope()); + EXPECT_FALSE(udp_packet_writer->isBatchMode()); +} + } // namespace } // namespace Server } // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index cf99f6b3f17a..40f0ea30ffce 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -90,6 +90,7 @@ ENV EOF EOS EOY +EPOLLOUT EPOLLRDHUP EQ ERANGE @@ -146,6 +147,7 @@ IDL IETF INADDR INET +INVAL IO IOS IP @@ -431,6 +433,7 @@ bools borks broadcasted buf +buflen bugprone builtin builtins @@ -633,6 +636,7 @@ gmock goog google goto +gso gzip hackery hacky From d58f7025f77ba4bff10ad48ea8685916f0cd52f1 Mon Sep 17 00:00:00 2001 From: Jonh Wendell Date: Sat, 8 Aug 2020 17:53:25 -0400 Subject: [PATCH 08/67] Bump golang to 1.14.7 (#12557) Signed-off-by: Jonh Wendell --- bazel/dependency_imports.bzl | 2 +- bazel/repository_locations.bzl | 4 ++-- ci/verify_examples.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index a0a0110f4735..56aa348f4be0 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -10,7 +10,7 @@ load("@protodoc_pip3//:requirements.bzl", protodoc_pip_install = "pip_install") load("@rules_antlr//antlr:deps.bzl", "antlr_dependencies") # go version for rules_go -GO_VERSION = "1.14.4" +GO_VERSION = "1.14.7" def envoy_dependency_imports(go_version = GO_VERSION): rules_foreign_cc_dependencies() diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 14c8b6d625be..73698991a2f4 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -354,8 +354,8 @@ DEPENDENCY_REPOSITORIES = dict( cpe = "N/A", ), io_bazel_rules_go = dict( - sha256 = "a8d6b1b354d371a646d2f7927319974e0f9e52f73a2452d2b3877118169eb6bb", - urls = ["https://github.com/bazelbuild/rules_go/releases/download/v0.23.3/rules_go-v0.23.3.tar.gz"], + sha256 = "0310e837aed522875791750de44408ec91046c630374990edd51827cb169f616", + urls = ["https://github.com/bazelbuild/rules_go/releases/download/v0.23.7/rules_go-v0.23.7.tar.gz"], use_category = ["build"], ), rules_cc = dict( diff --git a/ci/verify_examples.sh b/ci/verify_examples.sh index 61fb380ef2d6..711ceb5f25a3 100755 --- a/ci/verify_examples.sh +++ b/ci/verify_examples.sh @@ -23,7 +23,7 @@ cd ../ # Test grpc bridge example # install go -GO_VERSION="1.14.4" +GO_VERSION="1.14.7" curl -O https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz tar -xf go$GO_VERSION.linux-amd64.tar.gz sudo mv go /usr/local From 8c2019ab978820688ebba729bbd82d418d2be488 Mon Sep 17 00:00:00 2001 From: DongRyeol Cha Date: Mon, 10 Aug 2020 15:04:15 +0900 Subject: [PATCH 09/67] build: Fix that selective testing does not work (#12563) As you know that the envoy build system has a functionality that can choose some tests to do test manually. But now, to do that, I need to add a dummy argument when I execute the do_ci.sh like as following: /ci/run_envoy_docker.sh "./ci/do_ci.sh bazel.dev '' //test/common/network:udp_listener_impl_test". So, this patch fixes that selective testing works like as previous behavior. Signed-off-by: DongRyeol Cha --- ci/do_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 895ab71747d8..3218361ce81d 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -101,7 +101,7 @@ function bazel_binary_build() { CI_TARGET=$1 shift -if [[ $# -gt 1 ]]; then +if [[ $# -ge 1 ]]; then COVERAGE_TEST_TARGETS=$* TEST_TARGETS="$COVERAGE_TEST_TARGETS" else From bb3571a6207820ad0de4ef9be6db838e175f6054 Mon Sep 17 00:00:00 2001 From: Sam Flattery <44659644+samflattery@users.noreply.github.com> Date: Mon, 10 Aug 2020 13:28:29 +0100 Subject: [PATCH 10/67] xds: remove warming listeners in xDS SOTW updates (#12461) Commit Message: Remove warming listeners in xDS SOTW updates Additional Description: - my xDS fuzzer timed out on OSS fuzz here because Envoy does not remove warming listeners when they are left out of a SOTW update like it removes active ones - changes LdsApiImpl::onConfigUpdate to use a new function in ListenerManagerImpl, allListeners() instead of the old listeners() which just returns active listeners to compute the diff between the update and the current SOTW - changed lds_api_test.cc to use this new function Signed-off-by: Sam Flattery --- include/envoy/server/listener_manager.h | 30 +++++++-- source/server/lds_api.cc | 3 +- source/server/listener_manager_impl.cc | 27 ++++++-- source/server/listener_manager_impl.h | 3 +- test/integration/ads_integration_test.cc | 62 +++++++++++++++++++ test/mocks/server/listener_manager.h | 3 +- ...e-minimized-xds_fuzz_test-6524356210196480 | 54 ++++++++++++++++ test/server/hot_restarting_parent_test.cc | 6 +- test/server/lds_api_test.cc | 17 +++-- 9 files changed, 186 insertions(+), 19 deletions(-) create mode 100644 test/server/config_validation/xds_corpus/clusterfuzz-testcase-minimized-xds_fuzz_test-6524356210196480 diff --git a/include/envoy/server/listener_manager.h b/include/envoy/server/listener_manager.h index 956a89264ac4..e01551414def 100644 --- a/include/envoy/server/listener_manager.h +++ b/include/envoy/server/listener_manager.h @@ -131,6 +131,16 @@ class ListenerManager { All, }; + // The types of listeners to be returned from listeners(ListenerState). + // An enum instead of enum class so the underlying type is an int and bitwise operations can be + // used without casting. + enum ListenerState : uint8_t { + ACTIVE = 1 << 0, + WARMING = 1 << 1, + DRAINING = 1 << 2, + ALL = ACTIVE | WARMING | DRAINING + }; + virtual ~ListenerManager() = default; /** @@ -161,11 +171,15 @@ class ListenerManager { virtual void createLdsApi(const envoy::config::core::v3::ConfigSource& lds_config) PURE; /** - * @return std::vector> a list of the currently - * loaded listeners. Note that this routine returns references to the existing listeners. The - * references are only valid in the context of the current call stack and should not be stored. + * @param state the type of listener to be returned (defaults to ACTIVE), states can be OR'd + * together to return multiple different types + * @return std::vector> a list of currently known + * listeners in the requested state. Note that this routine returns references to the existing + * listeners. The references are only valid in the context of the current call stack and should + * not be stored. */ - virtual std::vector> listeners() PURE; + virtual std::vector> + listeners(ListenerState state = ListenerState::ACTIVE) PURE; /** * @return uint64_t the total number of connections owned by all listeners across all workers. @@ -223,5 +237,13 @@ class ListenerManager { virtual ApiListenerOptRef apiListener() PURE; }; +// overload operator| to allow ListenerManager::listeners(ListenerState) to be called using a +// combination of flags, such as listeners(ListenerState::WARMING|ListenerState::ACTIVE) +constexpr ListenerManager::ListenerState operator|(const ListenerManager::ListenerState lhs, + const ListenerManager::ListenerState rhs) { + return static_cast(static_cast(lhs) | + static_cast(rhs)); +} + } // namespace Server } // namespace Envoy diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 3165a1525ce6..4a1a65ed125b 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -97,7 +97,8 @@ void LdsApiImpl::onConfigUpdate(const std::vector& r // We need to keep track of which listeners need to remove. // Specifically, it's [listeners we currently have] - [listeners found in the response]. absl::node_hash_set listeners_to_remove; - for (const auto& listener : listener_manager_.listeners()) { + for (const auto& listener : + listener_manager_.listeners(ListenerManager::WARMING | ListenerManager::ACTIVE)) { listeners_to_remove.insert(listener.get().name()); } for (const auto& resource : resources) { diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index fa384c274fa2..a257b69b36e5 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -639,11 +639,30 @@ ListenerManagerImpl::getListenerByName(ListenerList& listeners, const std::strin return ret; } -std::vector> ListenerManagerImpl::listeners() { +std::vector> +ListenerManagerImpl::listeners(ListenerState state) { std::vector> ret; - ret.reserve(active_listeners_.size()); - for (const auto& listener : active_listeners_) { - ret.push_back(*listener); + + size_t size = 0; + size += state & WARMING ? warming_listeners_.size() : 0; + size += state & ACTIVE ? active_listeners_.size() : 0; + size += state & DRAINING ? draining_listeners_.size() : 0; + ret.reserve(size); + + if (state & WARMING) { + for (const auto& listener : warming_listeners_) { + ret.push_back(*listener); + } + } + if (state & ACTIVE) { + for (const auto& listener : active_listeners_) { + ret.push_back(*listener); + } + } + if (state & DRAINING) { + for (const auto& draining_listener : draining_listeners_) { + ret.push_back(*(draining_listener.listener_)); + } } return ret; } diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 36f9ccd1e7f8..106e1d629dc9 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -186,7 +186,8 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable> listeners() override; + std::vector> + listeners(ListenerState state = ListenerState::ACTIVE) override; uint64_t numConnections() const override; bool removeListener(const std::string& listener_name) override; void startWorkers(GuardDog& guard_dog) override; diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 021beac26885..bf413b9d91d5 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -420,6 +420,68 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { {"warming_cluster_2", "warming_cluster_1"}, {}, {})); } +// Validate that warming listeners are removed when left out of SOTW update. +TEST_P(AdsIntegrationTest, RemoveWarmingListener) { + initialize(); + + // Send initial configuration to start workers, validate we can process a request. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("cluster_0")}, + {buildCluster("cluster_0")}, {}, "1"); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"cluster_0"}, {"cluster_0"}, {})); + + sendDiscoveryResponse( + Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, + {buildClusterLoadAssignment("cluster_0")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, + {buildListener("listener_0", "route_config_0")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", + {"cluster_0"}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"route_config_0"}, {"route_config_0"}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().RouteConfiguration, {buildRouteConfig("route_config_0", "cluster_0")}, + {buildRouteConfig("route_config_0", "cluster_0")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "1", + {"route_config_0"}, {}, {})); + + test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); + makeSingleRequest(); + + // Send a listener without its route, so it will be added as warming. + sendDiscoveryResponse( + Config::TypeUrl::get().Listener, + {buildListener("listener_0", "route_config_0"), + buildListener("warming_listener_1", "nonexistent_route")}, + {buildListener("warming_listener_1", "nonexistent_route")}, {}, "2"); + test_server_->waitForGaugeEq("listener_manager.total_listeners_warming", 1); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "1", + {"nonexistent_route", "route_config_0"}, + {"nonexistent_route"}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "2", {}, {}, {})); + + // Send a request removing the warming listener. + sendDiscoveryResponse( + Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, + {buildListener("listener_0", "route_config_0")}, {"warming_listener_1"}, "3"); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "1", + {"route_config_0"}, {}, {"nonexistent_route"})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "3", {}, {}, {})); + + // The warming listener should be successfully removed. + test_server_->waitForCounterEq("listener_manager.listener_removed", 1); + test_server_->waitForGaugeEq("listener_manager.total_listeners_warming", 0); +} + // Verify cluster warming is finished only on named EDS response. TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { initialize(); diff --git a/test/mocks/server/listener_manager.h b/test/mocks/server/listener_manager.h index 889dfa1f521f..a91a9acb1764 100644 --- a/test/mocks/server/listener_manager.h +++ b/test/mocks/server/listener_manager.h @@ -15,7 +15,8 @@ class MockListenerManager : public ListenerManager { (const envoy::config::listener::v3::Listener& config, const std::string& version_info, bool modifiable)); MOCK_METHOD(void, createLdsApi, (const envoy::config::core::v3::ConfigSource& lds_config)); - MOCK_METHOD(std::vector>, listeners, ()); + MOCK_METHOD(std::vector>, listeners, + (ListenerState state)); MOCK_METHOD(uint64_t, numConnections, (), (const)); MOCK_METHOD(bool, removeListener, (const std::string& listener_name)); MOCK_METHOD(void, startWorkers, (GuardDog & guard_dog)); diff --git a/test/server/config_validation/xds_corpus/clusterfuzz-testcase-minimized-xds_fuzz_test-6524356210196480 b/test/server/config_validation/xds_corpus/clusterfuzz-testcase-minimized-xds_fuzz_test-6524356210196480 new file mode 100644 index 000000000000..df27fe2695e7 --- /dev/null +++ b/test/server/config_validation/xds_corpus/clusterfuzz-testcase-minimized-xds_fuzz_test-6524356210196480 @@ -0,0 +1,54 @@ +actions { + add_listener { + listener_num: 256 + route_num: 6356993 + } +} +actions { + add_listener { + route_num: 16 + } +} +actions { + remove_listener { + } +} +actions { + add_route { + route_num: 1 + } +} +actions { + add_listener { + route_num: 11264 + } +} +actions { + add_listener { + listener_num: 2147483648 + route_num: 2147483648 + } +} +actions { + add_listener { + listener_num: 6356993 + route_num: 11264 + } +} +actions { + add_listener { + listener_num: 256 + route_num: 65537 + } +} +actions { + add_route { + route_num: 1 + } +} +actions { + add_listener { + listener_num: 2147483648 + route_num: 2 + } +} diff --git a/test/server/hot_restarting_parent_test.cc b/test/server/hot_restarting_parent_test.cc index 80ce667bb50d..a3f405d550db 100644 --- a/test/server/hot_restarting_parent_test.cc +++ b/test/server/hot_restarting_parent_test.cc @@ -36,7 +36,8 @@ TEST_F(HotRestartingParentTest, GetListenSocketsForChildNotFound) { MockListenerManager listener_manager; std::vector> listeners; EXPECT_CALL(server_, listenerManager()).WillOnce(ReturnRef(listener_manager)); - EXPECT_CALL(listener_manager, listeners()).WillOnce(Return(listeners)); + EXPECT_CALL(listener_manager, listeners(ListenerManager::ListenerState::ACTIVE)) + .WillOnce(Return(listeners)); HotRestartMessage::Request request; request.mutable_pass_listen_socket()->set_address("tcp://127.0.0.1:80"); @@ -51,7 +52,8 @@ TEST_F(HotRestartingParentTest, GetListenSocketsForChildNotBindPort) { InSequence s; listeners.push_back(std::ref(*static_cast(&listener_config))); EXPECT_CALL(server_, listenerManager()).WillOnce(ReturnRef(listener_manager)); - EXPECT_CALL(listener_manager, listeners()).WillOnce(Return(listeners)); + EXPECT_CALL(listener_manager, listeners(ListenerManager::ListenerState::ACTIVE)) + .WillOnce(Return(listeners)); EXPECT_CALL(listener_config, listenSocketFactory()); EXPECT_CALL(listener_config.socket_factory_, localAddress()); EXPECT_CALL(listener_config, bindToPort()).WillOnce(Return(false)); diff --git a/test/server/lds_api_test.cc b/test/server/lds_api_test.cc index f4c5aee7b72c..54a84886e832 100644 --- a/test/server/lds_api_test.cc +++ b/test/server/lds_api_test.cc @@ -76,7 +76,8 @@ class LdsApiTest : public testing::Test { listeners_.back().name_ = name; refs.emplace_back(listeners_.back()); } - EXPECT_CALL(listener_manager_, listeners()).WillOnce(Return(refs)); + EXPECT_CALL(listener_manager_, listeners(ListenerManager::WARMING | ListenerManager::ACTIVE)) + .WillOnce(Return(refs)); EXPECT_CALL(listener_manager_, beginListenerUpdate()); } @@ -120,7 +121,8 @@ TEST_F(LdsApiTest, MisconfiguredListenerNameIsPresentInException) { socket_address->set_port_value(1); listener.add_filter_chains(); - EXPECT_CALL(listener_manager_, listeners()).WillOnce(Return(existing_listeners)); + EXPECT_CALL(listener_manager_, listeners(ListenerManager::WARMING | ListenerManager::ACTIVE)) + .WillOnce(Return(existing_listeners)); EXPECT_CALL(listener_manager_, beginListenerUpdate()); EXPECT_CALL(listener_manager_, addOrUpdateListener(_, _, true)) @@ -141,7 +143,8 @@ TEST_F(LdsApiTest, EmptyListenersUpdate) { std::vector> existing_listeners; - EXPECT_CALL(listener_manager_, listeners()).WillOnce(Return(existing_listeners)); + EXPECT_CALL(listener_manager_, listeners(ListenerManager::WARMING | ListenerManager::ACTIVE)) + .WillOnce(Return(existing_listeners)); EXPECT_CALL(listener_manager_, beginListenerUpdate()); EXPECT_CALL(listener_manager_, endListenerUpdate(_)) .WillOnce(Invoke([](ListenerManager::FailureStates&& state) { EXPECT_EQ(0, state.size()); })); @@ -164,7 +167,8 @@ TEST_F(LdsApiTest, ListenerCreationContinuesEvenAfterException) { const auto listener_2 = buildListener("valid-listener-2"); const auto listener_3 = buildListener("invalid-listener-2"); - EXPECT_CALL(listener_manager_, listeners()).WillOnce(Return(existing_listeners)); + EXPECT_CALL(listener_manager_, listeners(ListenerManager::WARMING | ListenerManager::ACTIVE)) + .WillOnce(Return(existing_listeners)); EXPECT_CALL(listener_manager_, beginListenerUpdate()); EXPECT_CALL(listener_manager_, addOrUpdateListener(_, _, true)) @@ -195,7 +199,8 @@ TEST_F(LdsApiTest, ValidateDuplicateListeners) { const auto listener = buildListener("duplicate_listener"); std::vector> existing_listeners; - EXPECT_CALL(listener_manager_, listeners()).WillOnce(Return(existing_listeners)); + EXPECT_CALL(listener_manager_, listeners(ListenerManager::WARMING | ListenerManager::ACTIVE)) + .WillOnce(Return(existing_listeners)); EXPECT_CALL(listener_manager_, beginListenerUpdate()); EXPECT_CALL(listener_manager_, addOrUpdateListener(_, _, true)).WillOnce(Return(true)); EXPECT_CALL(listener_manager_, endListenerUpdate(_)); @@ -347,7 +352,7 @@ version_info: '1' address: tcp://0.0.0.1 port_value: 61000 filter_chains: - - filters: + - filters: )EOF"; auto response1 = TestUtility::parseYaml(response1_yaml); From 2187f1070a76a124a4d7cebe72551804b1a5218b Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 10 Aug 2020 08:56:05 -0400 Subject: [PATCH 11/67] http1: remove exceptions from H/1 codec (#11778) Commit Message: Remove all throw statements from H/1 codec This change removed all uses of C++ exceptions from H/1 codec. I modeled the flow after Yan's H/2 work (#11575). Codec status are set in uniform helper methods. This is the only change from the previous PR (#11101), besides merging newer exceptions. This change replaces all throw statements with a return of corresponding error Status and adds plumbing to return the status to codec callers. The dispatch() method returns the encountered error to the caller, which will be handled accordingly. The calls to the RequestEncoder::encodeHeaders() NOT called from dispatch() method will RELEASE_ASSERT if an error code is returned. This does not alter the existing behavior of abnormally terminating the process, just the method of termination: RELEASE_ASSERT vs uncaught exception. Risk Level: High (Codec changes) Signed-off-by: Asra Ali --- source/common/http/http1/BUILD | 2 +- source/common/http/http1/codec_impl.cc | 253 ++++++++++++++-------- source/common/http/http1/codec_impl.h | 88 +++++--- test/common/http/codec_impl_fuzz_test.cc | 2 +- test/common/http/http1/codec_impl_test.cc | 39 +++- 5 files changed, 250 insertions(+), 134 deletions(-) diff --git a/source/common/http/http1/BUILD b/source/common/http/http1/BUILD index 9451c4e29ae3..2fb4325d9810 100644 --- a/source/common/http/http1/BUILD +++ b/source/common/http/http1/BUILD @@ -55,7 +55,7 @@ envoy_cc_library( srcs = ["codec_impl.cc"], hdrs = ["codec_impl.h"], external_deps = ["http_parser"], - deps = CODEC_LIB_DEPS, + deps = CODEC_LIB_DEPS + ["//source/common/common:cleanup_lib"], ) envoy_cc_library( diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index c9cf88f569e8..4e8c54ed7db5 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -9,7 +9,9 @@ #include "envoy/http/header_map.h" #include "envoy/network/connection.h" +#include "common/common/cleanup.h" #include "common/common/enum_to_int.h" +#include "common/common/statusor.h" #include "common/common/utility.h" #include "common/grpc/common.h" #include "common/http/exception.h" @@ -275,9 +277,10 @@ void ServerConnectionImpl::maybeAddSentinelBufferFragment(Buffer::WatermarkBuffe outbound_responses_++; } -void ServerConnectionImpl::doFloodProtectionChecks() const { +Status ServerConnectionImpl::doFloodProtectionChecks() const { + ASSERT(dispatching_); if (!flood_protection_) { - return; + return okStatus(); } // Before processing another request, make sure that we are below the response flood protection // threshold. @@ -285,8 +288,9 @@ void ServerConnectionImpl::doFloodProtectionChecks() const { ENVOY_CONN_LOG(trace, "error accepting request: too many pending responses queued", connection_); stats_.response_flood_.inc(); - throw FrameFloodException("Too many responses queued."); + return bufferFloodError("Too many responses queued."); } + return okStatus(); } void ConnectionImpl::flushOutput(bool end_encode) { @@ -372,12 +376,14 @@ void RequestEncoderImpl::encodeHeaders(const RequestHeaderMap& headers, bool end const HeaderEntry* host = headers.Host(); bool is_connect = HeaderUtility::isConnect(headers); - if (!method || (!path && !is_connect)) { - // TODO(#10878): This exception does not occur during dispatch and would not be triggered under - // normal circumstances since inputs would fail parsing at ingress. Replace with proper error - // handling when exceptions are removed. Include missing host header for CONNECT. - throw CodecClientException(":method and :path must be specified"); - } + // TODO(#10878): Include missing host header for CONNECT. + // The RELEASE_ASSERT below does not change the existing behavior of `encodeHeaders`. + // The `encodeHeaders` used to throw on errors. Callers of `encodeHeaders()` do not catch + // exceptions and this would cause abnormal process termination in error cases. This change + // replaces abnormal process termination from unhandled exception with the RELEASE_ASSERT. Further + // work will replace this RELEASE_ASSERT with proper error handling. + RELEASE_ASSERT(method && (path || is_connect), ":method and :path must be specified"); + if (method->value() == Headers::get().MethodValues.Head) { head_request_ = true; } else if (method->value() == Headers::get().MethodValues.Connect) { @@ -400,34 +406,57 @@ void RequestEncoderImpl::encodeHeaders(const RequestHeaderMap& headers, bool end encodeHeadersBase(headers, absl::nullopt, end_stream); } +int ConnectionImpl::setAndCheckCallbackStatus(Status&& status) { + ASSERT(codec_status_.ok()); + codec_status_ = std::move(status); + return codec_status_.ok() ? enumToInt(HttpParserCode::Success) : enumToInt(HttpParserCode::Error); +} + +int ConnectionImpl::setAndCheckCallbackStatusOr(Envoy::StatusOr&& statusor) { + ASSERT(codec_status_.ok()); + if (statusor.ok()) { + return statusor.value(); + } else { + codec_status_ = std::move(statusor.status()); + return enumToInt(HttpParserCode::Error); + } +} + http_parser_settings ConnectionImpl::settings_{ [](http_parser* parser) -> int { - static_cast(parser->data)->onMessageBeginBase(); - return 0; + auto* conn_impl = static_cast(parser->data); + auto status = conn_impl->onMessageBeginBase(); + return conn_impl->setAndCheckCallbackStatus(std::move(status)); }, [](http_parser* parser, const char* at, size_t length) -> int { - static_cast(parser->data)->onUrl(at, length); - return 0; + auto* conn_impl = static_cast(parser->data); + auto status = conn_impl->onUrl(at, length); + return conn_impl->setAndCheckCallbackStatus(std::move(status)); }, nullptr, // on_status [](http_parser* parser, const char* at, size_t length) -> int { - static_cast(parser->data)->onHeaderField(at, length); - return 0; + auto* conn_impl = static_cast(parser->data); + auto status = conn_impl->onHeaderField(at, length); + return conn_impl->setAndCheckCallbackStatus(std::move(status)); }, [](http_parser* parser, const char* at, size_t length) -> int { - static_cast(parser->data)->onHeaderValue(at, length); - return 0; + auto* conn_impl = static_cast(parser->data); + auto status = conn_impl->onHeaderValue(at, length); + return conn_impl->setAndCheckCallbackStatus(std::move(status)); }, [](http_parser* parser) -> int { - return static_cast(parser->data)->onHeadersCompleteBase(); + auto* conn_impl = static_cast(parser->data); + auto statusor = conn_impl->onHeadersCompleteBase(); + return conn_impl->setAndCheckCallbackStatusOr(std::move(statusor)); }, [](http_parser* parser, const char* at, size_t length) -> int { static_cast(parser->data)->bufferBody(at, length); return 0; }, [](http_parser* parser) -> int { - static_cast(parser->data)->onMessageCompleteBase(); - return 0; + auto* conn_impl = static_cast(parser->data); + auto status = conn_impl->onMessageCompleteBase(); + return conn_impl->setAndCheckCallbackStatus(std::move(status)); }, [](http_parser* parser) -> int { // A 0-byte chunk header is used to signal the end of the chunked body. @@ -453,6 +482,7 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat enable_trailers_(enable_trailers), strict_1xx_and_204_headers_(Runtime::runtimeFeatureEnabled( "envoy.reloadable_features.strict_1xx_and_204_response_headers")), + dispatching_(false), output_buffer_([&]() -> void { this->onBelowLowWatermark(); }, [&]() -> void { this->onAboveHighWatermark(); }, []() -> void { /* TODO(adisuissa): Handle overflow watermark */ }), @@ -462,11 +492,12 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat parser_.data = this; } -void ConnectionImpl::completeLastHeader() { +Status ConnectionImpl::completeLastHeader() { + ASSERT(dispatching_); ENVOY_CONN_LOG(trace, "completed header: key={} value={}", connection_, current_header_field_.getStringView(), current_header_value_.getStringView()); - checkHeaderNameForUnderscores(); + RETURN_IF_ERROR(checkHeaderNameForUnderscores()); auto& headers_or_trailers = headersOrTrailers(); if (!current_header_field_.empty()) { current_header_field_.inlineTransform([](char c) { return absl::ascii_tolower(c); }); @@ -481,15 +512,16 @@ void ConnectionImpl::completeLastHeader() { // Check if the number of headers exceeds the limit. if (headers_or_trailers.size() > max_headers_count_) { error_code_ = Http::Code::RequestHeaderFieldsTooLarge; - sendProtocolError(Http1ResponseCodeDetails::get().TooManyHeaders); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().TooManyHeaders)); const absl::string_view header_type = processing_trailers_ ? Http1HeaderTypes::get().Trailers : Http1HeaderTypes::get().Headers; - throw CodecProtocolException(absl::StrCat(header_type, " size exceeds limit")); + return codecProtocolError(absl::StrCat(header_type, " size exceeds limit")); } header_parsing_state_ = HeaderParsingState::Field; ASSERT(current_header_field_.empty()); ASSERT(current_header_value_.empty()); + return okStatus(); } uint32_t ConnectionImpl::getHeadersSize() { @@ -497,15 +529,16 @@ uint32_t ConnectionImpl::getHeadersSize() { headersOrTrailers().byteSize(); } -void ConnectionImpl::checkMaxHeadersSize() { +Status ConnectionImpl::checkMaxHeadersSize() { const uint32_t total = getHeadersSize(); if (total > (max_headers_kb_ * 1024)) { const absl::string_view header_type = processing_trailers_ ? Http1HeaderTypes::get().Trailers : Http1HeaderTypes::get().Headers; error_code_ = Http::Code::RequestHeaderFieldsTooLarge; - sendProtocolError(Http1ResponseCodeDetails::get().HeadersTooLarge); - throw CodecProtocolException(absl::StrCat(header_type, " size exceeds limit")); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().HeadersTooLarge)); + return codecProtocolError(absl::StrCat(header_type, " size exceeds limit")); } + return okStatus(); } bool ConnectionImpl::maybeDirectDispatch(Buffer::Instance& data) { @@ -530,8 +563,14 @@ Http::Status ConnectionImpl::dispatch(Buffer::Instance& data) { Http::Status ConnectionImpl::innerDispatch(Buffer::Instance& data) { ENVOY_CONN_LOG(trace, "parsing {} bytes", connection_, data.length()); + // Make sure that dispatching_ is set to false after dispatching, even when + // http_parser exits early with an error code. + Cleanup cleanup([this]() { dispatching_ = false; }); + ASSERT(!dispatching_); + ASSERT(codec_status_.ok()); ASSERT(buffered_body_.length() == 0); + dispatching_ = true; if (maybeDirectDispatch(data)) { return Http::okStatus(); } @@ -542,7 +581,11 @@ Http::Status ConnectionImpl::innerDispatch(Buffer::Instance& data) { ssize_t total_parsed = 0; if (data.length() > 0) { for (const Buffer::RawSlice& slice : data.getRawSlices()) { - total_parsed += dispatchSlice(static_cast(slice.mem_), slice.len_); + auto statusor_parsed = dispatchSlice(static_cast(slice.mem_), slice.len_); + if (!statusor_parsed.ok()) { + return statusor_parsed.status(); + } + total_parsed += statusor_parsed.value(); if (HTTP_PARSER_ERRNO(&parser_) != HPE_OK) { // Parse errors trigger an exception in dispatchSlice so we are guaranteed to be paused at // this point. @@ -552,7 +595,10 @@ Http::Status ConnectionImpl::innerDispatch(Buffer::Instance& data) { } dispatchBufferedBody(); } else { - dispatchSlice(nullptr, 0); + auto result = dispatchSlice(nullptr, 0); + if (!result.ok()) { + return result.status(); + } } ASSERT(buffered_body_.length() == 0); @@ -565,50 +611,59 @@ Http::Status ConnectionImpl::innerDispatch(Buffer::Instance& data) { return Http::okStatus(); } -size_t ConnectionImpl::dispatchSlice(const char* slice, size_t len) { +Envoy::StatusOr ConnectionImpl::dispatchSlice(const char* slice, size_t len) { + ASSERT(codec_status_.ok() && dispatching_); ssize_t rc = http_parser_execute(&parser_, &settings_, slice, len); + if (!codec_status_.ok()) { + return codec_status_; + } if (HTTP_PARSER_ERRNO(&parser_) != HPE_OK && HTTP_PARSER_ERRNO(&parser_) != HPE_PAUSED) { - sendProtocolError(Http1ResponseCodeDetails::get().HttpCodecError); - throw CodecProtocolException("http/1.1 protocol error: " + - std::string(http_errno_name(HTTP_PARSER_ERRNO(&parser_)))); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().HttpCodecError)); + // Avoid overwriting the codec_status_ set in the callbacks. + ASSERT(codec_status_.ok()); + codec_status_ = codecProtocolError( + absl::StrCat("http/1.1 protocol error: ", http_errno_name(HTTP_PARSER_ERRNO(&parser_)))); + return codec_status_; } return rc; } -void ConnectionImpl::onHeaderField(const char* data, size_t length) { +Status ConnectionImpl::onHeaderField(const char* data, size_t length) { + ASSERT(dispatching_); // We previously already finished up the headers, these headers are // now trailers. if (header_parsing_state_ == HeaderParsingState::Done) { if (!enable_trailers_) { // Ignore trailers. - return; + return okStatus(); } processing_trailers_ = true; header_parsing_state_ = HeaderParsingState::Field; allocTrailers(); } if (header_parsing_state_ == HeaderParsingState::Value) { - completeLastHeader(); + RETURN_IF_ERROR(completeLastHeader()); } current_header_field_.append(data, length); - checkMaxHeadersSize(); + return checkMaxHeadersSize(); } -void ConnectionImpl::onHeaderValue(const char* data, size_t length) { +Status ConnectionImpl::onHeaderValue(const char* data, size_t length) { + ASSERT(dispatching_); if (header_parsing_state_ == HeaderParsingState::Done && !enable_trailers_) { // Ignore trailers. - return; + return okStatus(); } absl::string_view header_value{data, length}; if (!Http::HeaderUtility::headerValueIsValid(header_value)) { ENVOY_CONN_LOG(debug, "invalid header value: {}", connection_, header_value); error_code_ = Http::Code::BadRequest; - sendProtocolError(Http1ResponseCodeDetails::get().InvalidCharacters); - throw CodecProtocolException("http/1.1 protocol error: header value contains invalid chars"); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().InvalidCharacters)); + return codecProtocolError("http/1.1 protocol error: header value contains invalid chars"); } header_parsing_state_ = HeaderParsingState::Value; @@ -621,13 +676,14 @@ void ConnectionImpl::onHeaderValue(const char* data, size_t length) { } current_header_value_.append(header_value.data(), header_value.length()); - checkMaxHeadersSize(); + return checkMaxHeadersSize(); } -int ConnectionImpl::onHeadersCompleteBase() { +Envoy::StatusOr ConnectionImpl::onHeadersCompleteBase() { ASSERT(!processing_trailers_); + ASSERT(dispatching_); ENVOY_CONN_LOG(trace, "onHeadersCompleteBase", connection_); - completeLastHeader(); + RETURN_IF_ERROR(completeLastHeader()); if (!(parser_.http_major == 1 && parser_.http_minor == 1)) { // This is not necessarily true, but it's good enough since higher layers only care if this is @@ -666,8 +722,8 @@ int ConnectionImpl::onHeadersCompleteBase() { // Per https://tools.ietf.org/html/rfc7231#section-4.3.6 a payload with a // CONNECT request has no defined semantics, and may be rejected. error_code_ = Http::Code::BadRequest; - sendProtocolError(Http1ResponseCodeDetails::get().BodyDisallowed); - throw CodecProtocolException("http/1.1 protocol error: unsupported content length"); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().BodyDisallowed)); + return codecProtocolError("http/1.1 protocol error: unsupported content length"); } } ENVOY_CONN_LOG(trace, "codec entering upgrade mode for CONNECT request.", connection_); @@ -683,16 +739,20 @@ int ConnectionImpl::onHeadersCompleteBase() { if (!absl::EqualsIgnoreCase(encoding, Headers::get().TransferEncodingValues.Chunked) || parser_.method == HTTP_CONNECT) { error_code_ = Http::Code::NotImplemented; - sendProtocolError(Http1ResponseCodeDetails::get().InvalidTransferEncoding); - throw CodecProtocolException("http/1.1 protocol error: unsupported transfer encoding"); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().InvalidTransferEncoding)); + return codecProtocolError("http/1.1 protocol error: unsupported transfer encoding"); } } - int rc = onHeadersComplete(); + auto statusor = onHeadersComplete(); + if (!statusor.ok()) { + RETURN_IF_ERROR(statusor.status()); + } + header_parsing_state_ = HeaderParsingState::Done; // Returning 2 informs http_parser to not expect a body or further data on this connection. - return handling_upgrade_ ? 2 : rc; + return handling_upgrade_ ? 2 : statusor.value(); } void ConnectionImpl::bufferBody(const char* data, size_t length) { @@ -701,6 +761,7 @@ void ConnectionImpl::bufferBody(const char* data, size_t length) { void ConnectionImpl::dispatchBufferedBody() { ASSERT(HTTP_PARSER_ERRNO(&parser_) == HPE_OK || HTTP_PARSER_ERRNO(&parser_) == HPE_PAUSED); + ASSERT(codec_status_.ok()); if (buffered_body_.length() > 0) { onBody(buffered_body_); buffered_body_.drain(buffered_body_.length()); @@ -715,7 +776,7 @@ void ConnectionImpl::onChunkHeader(bool is_final_chunk) { } } -void ConnectionImpl::onMessageCompleteBase() { +Status ConnectionImpl::onMessageCompleteBase() { ENVOY_CONN_LOG(trace, "message complete", connection_); dispatchBufferedBody(); @@ -726,19 +787,20 @@ void ConnectionImpl::onMessageCompleteBase() { ASSERT(!deferred_end_stream_headers_); ENVOY_CONN_LOG(trace, "Pausing parser due to upgrade.", connection_); http_parser_pause(&parser_, 1); - return; + return okStatus(); } // If true, this indicates we were processing trailers and must // move the last header into current_header_map_ if (header_parsing_state_ == HeaderParsingState::Value) { - completeLastHeader(); + RETURN_IF_ERROR(completeLastHeader()); } onMessageComplete(); + return okStatus(); } -void ConnectionImpl::onMessageBeginBase() { +Status ConnectionImpl::onMessageBeginBase() { ENVOY_CONN_LOG(trace, "message begin", connection_); // Make sure that if HTTP/1.0 and HTTP/1.1 requests share a connection Envoy correctly sets // protocol for each request. Envoy defaults to 1.1 but sets the protocol to 1.0 where applicable @@ -747,7 +809,7 @@ void ConnectionImpl::onMessageBeginBase() { processing_trailers_ = false; header_parsing_state_ = HeaderParsingState::Field; allocHeaders(); - onMessageBegin(); + return onMessageBegin(); } void ConnectionImpl::onResetStreamBase(StreamResetReason reason) { @@ -795,7 +857,7 @@ void ServerConnectionImpl::onEncodeComplete() { } } -void ServerConnectionImpl::handlePath(RequestHeaderMap& headers, unsigned int method) { +Status ServerConnectionImpl::handlePath(RequestHeaderMap& headers, unsigned int method) { HeaderString path(Headers::get().Path); bool is_connect = (method == HTTP_CONNECT); @@ -806,7 +868,7 @@ void ServerConnectionImpl::handlePath(RequestHeaderMap& headers, unsigned int me (active_request.request_url_.getStringView()[0] == '/' || ((method == HTTP_OPTIONS) && active_request.request_url_.getStringView()[0] == '*'))) { headers.addViaMove(std::move(path), std::move(active_request.request_url_)); - return; + return okStatus(); } // If absolute_urls and/or connect are not going be handled, copy the url and return. @@ -815,13 +877,13 @@ void ServerConnectionImpl::handlePath(RequestHeaderMap& headers, unsigned int me // Absolute URLS in CONNECT requests will be rejected below by the URL class validation. if (!codec_settings_.allow_absolute_url_ && !is_connect) { headers.addViaMove(std::move(path), std::move(active_request.request_url_)); - return; + return okStatus(); } Utility::Url absolute_url; if (!absolute_url.initialize(active_request.request_url_.getStringView(), is_connect)) { - sendProtocolError(Http1ResponseCodeDetails::get().InvalidUrl); - throw CodecProtocolException("http/1.1 protocol error: invalid url in request line"); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().InvalidUrl)); + return codecProtocolError("http/1.1 protocol error: invalid url in request line"); } // RFC7230#5.7 // When a proxy receives a request with an absolute-form of @@ -836,9 +898,10 @@ void ServerConnectionImpl::handlePath(RequestHeaderMap& headers, unsigned int me headers.setPath(absolute_url.pathAndQueryParams()); } active_request.request_url_.clear(); + return okStatus(); } -int ServerConnectionImpl::onHeadersComplete() { +Envoy::StatusOr ServerConnectionImpl::onHeadersComplete() { // Handle the case where response happens prior to request complete. It's up to upper layer code // to disconnect the connection but we shouldn't fire any more events since it doesn't make // sense. @@ -855,8 +918,9 @@ int ServerConnectionImpl::onHeadersComplete() { ENVOY_CONN_LOG(debug, "Invalid nominated headers in Connection: {}", connection_, header_value); error_code_ = Http::Code::BadRequest; - sendProtocolError(Http1ResponseCodeDetails::get().ConnectionHeaderSanitization); - throw CodecProtocolException("Invalid nominated headers in Connection."); + RETURN_IF_ERROR( + sendProtocolError(Http1ResponseCodeDetails::get().ConnectionHeaderSanitization)); + return codecProtocolError("Invalid nominated headers in Connection."); } } @@ -865,7 +929,7 @@ int ServerConnectionImpl::onHeadersComplete() { active_request.response_encoder_.setIsResponseToHeadRequest(parser_.method == HTTP_HEAD); active_request.response_encoder_.setIsResponseToConnectRequest(parser_.method == HTTP_CONNECT); - handlePath(*headers, parser_.method); + RETURN_IF_ERROR(handlePath(*headers, parser_.method)); ASSERT(active_request.request_url_.empty()); headers->setMethod(method_string); @@ -873,8 +937,8 @@ int ServerConnectionImpl::onHeadersComplete() { // Make sure the host is valid. auto details = HeaderUtility::requestHeadersValid(*headers); if (details.has_value()) { - sendProtocolError(details.value().get()); - throw CodecProtocolException( + RETURN_IF_ERROR(sendProtocolError(details.value().get())); + return codecProtocolError( "http/1.1 protocol error: request headers failed spec compliance checks"); } @@ -901,26 +965,31 @@ int ServerConnectionImpl::onHeadersComplete() { return 0; } -void ServerConnectionImpl::onMessageBegin() { +Status ServerConnectionImpl::onMessageBegin() { if (!resetStreamCalled()) { ASSERT(!active_request_.has_value()); active_request_.emplace(*this, header_key_formatter_.get()); auto& active_request = active_request_.value(); + if (resetStreamCalled()) { + return codecClientError("cannot create new streams after calling reset"); + } active_request.request_decoder_ = &callbacks_.newStream(active_request.response_encoder_); // Check for pipelined request flood as we prepare to accept a new request. // Parse errors that happen prior to onMessageBegin result in stream termination, it is not // possible to overflow output buffers with early parse errors. - doFloodProtectionChecks(); + RETURN_IF_ERROR(doFloodProtectionChecks()); } + return okStatus(); } -void ServerConnectionImpl::onUrl(const char* data, size_t length) { +Status ServerConnectionImpl::onUrl(const char* data, size_t length) { if (active_request_.has_value()) { active_request_.value().request_url_.append(data, length); - checkMaxHeadersSize(); + RETURN_IF_ERROR(checkMaxHeadersSize()); } + return okStatus(); } void ServerConnectionImpl::onBody(Buffer::Instance& data) { @@ -985,14 +1054,14 @@ void ServerConnectionImpl::sendProtocolErrorOld(absl::string_view details) { } } -void ServerConnectionImpl::sendProtocolError(absl::string_view details) { +Status ServerConnectionImpl::sendProtocolError(absl::string_view details) { if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.early_errors_via_hcm")) { sendProtocolErrorOld(details); - return; + return okStatus(); } // We do this here because we may get a protocol error before we have a logical stream. if (!active_request_.has_value()) { - onMessageBeginBase(); + RETURN_IF_ERROR(onMessageBeginBase()); } ASSERT(active_request_.has_value()); @@ -1009,8 +1078,8 @@ void ServerConnectionImpl::sendProtocolError(absl::string_view details) { active_request_->request_decoder_->sendLocalReply(is_grpc_request, error_code_, CodeUtility::toString(error_code_), nullptr, absl::nullopt, details); - return; } + return okStatus(); } void ServerConnectionImpl::onAboveHighWatermark() { @@ -1031,7 +1100,7 @@ void ServerConnectionImpl::releaseOutboundResponse( delete fragment; } -void ServerConnectionImpl::checkHeaderNameForUnderscores() { +Status ServerConnectionImpl::checkHeaderNameForUnderscores() { if (headers_with_underscores_action_ != envoy::config::core::v3::HttpProtocolOptions::ALLOW && Http::HeaderUtility::headerNameContainsUnderscore(current_header_field_.getStringView())) { if (headers_with_underscores_action_ == @@ -1045,11 +1114,12 @@ void ServerConnectionImpl::checkHeaderNameForUnderscores() { ENVOY_CONN_LOG(debug, "Rejecting request due to header name with underscores: {}", connection_, current_header_field_.getStringView()); error_code_ = Http::Code::BadRequest; - sendProtocolError(Http1ResponseCodeDetails::get().InvalidUnderscore); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().InvalidUnderscore)); stats_.requests_rejected_with_underscores_in_headers_.inc(); - throw CodecProtocolException("http/1.1 protocol error: header name contains underscores"); + return codecProtocolError("http/1.1 protocol error: header name contains underscores"); } } + return okStatus(); } ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, CodecStats& stats, @@ -1071,10 +1141,6 @@ bool ClientConnectionImpl::cannotHaveBody() { } RequestEncoder& ClientConnectionImpl::newStream(ResponseDecoder& response_decoder) { - if (resetStreamCalled()) { - throw CodecClientException("cannot create new streams after calling reset"); - } - // If reads were disabled due to flow control, we expect reads to always be enabled again before // reusing this connection. This is done when the response is received. ASSERT(connection_.readEnabled()); @@ -1086,14 +1152,14 @@ RequestEncoder& ClientConnectionImpl::newStream(ResponseDecoder& response_decode return pending_response_.value().encoder_; } -int ClientConnectionImpl::onHeadersComplete() { +Envoy::StatusOr ClientConnectionImpl::onHeadersComplete() { ENVOY_CONN_LOG(trace, "status_code {}", connection_, parser_.status_code); // Handle the case where the client is closing a kept alive connection (by sending a 408 // with a 'Connection: close' header). In this case we just let response flush out followed // by the remote close. if (!pending_response_.has_value() && !resetStreamCalled()) { - throw PrematureResponseException(static_cast(parser_.status_code)); + return prematureResponseError("", static_cast(parser_.status_code)); } else if (pending_response_.has_value()) { ASSERT(!pending_response_done_); auto& headers = absl::get(headers_or_trailers_); @@ -1110,23 +1176,25 @@ int ClientConnectionImpl::onHeadersComplete() { if (headers->TransferEncoding() && absl::EqualsIgnoreCase(headers->TransferEncoding()->value().getStringView(), Headers::get().TransferEncodingValues.Chunked)) { - sendProtocolError(Http1ResponseCodeDetails::get().InvalidTransferEncoding); - throw CodecProtocolException("http/1.1 protocol error: unsupported transfer encoding"); + RETURN_IF_ERROR(sendProtocolError(Http1ResponseCodeDetails::get().InvalidTransferEncoding)); + return codecProtocolError("http/1.1 protocol error: unsupported transfer encoding"); } } if (strict_1xx_and_204_headers_ && (parser_.status_code < 200 || parser_.status_code == 204)) { if (headers->TransferEncoding()) { - sendProtocolError(Http1ResponseCodeDetails::get().TransferEncodingNotAllowed); - throw CodecProtocolException( + RETURN_IF_ERROR( + sendProtocolError(Http1ResponseCodeDetails::get().TransferEncodingNotAllowed)); + return codecProtocolError( "http/1.1 protocol error: transfer encoding not allowed in 1xx or 204"); } if (headers->ContentLength()) { // Report a protocol error for non-zero Content-Length, but paper over zero Content-Length. if (headers->ContentLength()->value().getStringView() != "0") { - sendProtocolError(Http1ResponseCodeDetails::get().ContentLengthNotAllowed); - throw CodecProtocolException( + RETURN_IF_ERROR( + sendProtocolError(Http1ResponseCodeDetails::get().ContentLengthNotAllowed)); + return codecProtocolError( "http/1.1 protocol error: content length not allowed in 1xx or 204"); } @@ -1154,8 +1222,8 @@ int ClientConnectionImpl::onHeadersComplete() { } } - // Here we deal with cases where the response cannot have a body, but http_parser does not deal - // with it for us. + // Here we deal with cases where the response cannot have a body by returning 1, but http_parser + // does not deal with it for us. return cannotHaveBody() ? 1 : 0; } @@ -1215,11 +1283,12 @@ void ClientConnectionImpl::onResetStream(StreamResetReason reason) { } } -void ClientConnectionImpl::sendProtocolError(absl::string_view details) { +Status ClientConnectionImpl::sendProtocolError(absl::string_view details) { if (pending_response_.has_value()) { ASSERT(!pending_response_done_); pending_response_.value().encoder_.setDetails(details); } + return okStatus(); } void ClientConnectionImpl::onAboveHighWatermark() { diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index c74c0adae87c..0f8b5d7de71a 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -217,13 +217,38 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable&& statusor); + + // Codec errors found in callbacks are overridden within the http_parser library. This holds those + // errors to propagate them through to dispatch() where we can handle the error. + Envoy::Http::Status codec_status_; + protected: ConnectionImpl(Network::Connection& connection, CodecStats& stats, http_parser_type type, uint32_t max_headers_kb, const uint32_t max_headers_count, HeaderKeyFormatterPtr&& header_key_formatter, bool enable_trailers); + // The following define special return values for http_parser callbacks. See: + // https://github.com/nodejs/http-parser/blob/5c5b3ac62662736de9e71640a8dc16da45b32503/http_parser.h#L72 + // These codes do not overlap with standard HTTP Status codes. They are only used for user + // callbacks. + enum class HttpParserCode { + // Callbacks other than on_headers_complete should return a non-zero int to indicate an error + // and + // halt execution. + Error = -1, + Success = 0, + // Returning '1' from on_headers_complete will tell http_parser that it should not expect a + // body. + NoBody = 1, + // Returning '2' from on_headers_complete will tell http_parser that it should not expect a body + // nor any further data on the connection. + NoBodyData = 2, + }; + bool resetStreamCalled() { return reset_stream_called_; } - void onMessageBeginBase(); + Status onMessageBeginBase(); /** * Get memory used to represent HTTP headers or trailers currently being parsed. @@ -234,10 +259,10 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable dispatchSlice(const char* slice, size_t len); /** * Called by the http_parser when body data is received. @@ -314,37 +341,39 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable onHeadersCompleteBase(); + virtual Envoy::StatusOr onHeadersComplete() PURE; /** * Called to see if upgrade transition is allowed. @@ -365,8 +394,9 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable onHeadersComplete() override; // If upgrade behavior is not allowed, the HCM will have sanitized the headers out. bool upgradeAllowed() const override { return true; } void onBody(Buffer::Instance& data) override; void onResetStream(StreamResetReason reason) override; - void sendProtocolError(absl::string_view details) override; + Status sendProtocolError(absl::string_view details) override; void onAboveHighWatermark() override; void onBelowLowWatermark() override; HeaderMap& headersOrTrailers() override { @@ -495,8 +527,8 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { void releaseOutboundResponse(const Buffer::OwnedBufferFragmentImpl* fragment); void maybeAddSentinelBufferFragment(Buffer::WatermarkBuffer& output_buffer) override; - void doFloodProtectionChecks() const; - void checkHeaderNameForUnderscores() override; + Status doFloodProtectionChecks() const; + Status checkHeaderNameForUnderscores() override; ServerConnectionCallbacks& callbacks_; absl::optional active_request_; @@ -545,14 +577,14 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl { // ConnectionImpl void onEncodeComplete() override {} - void onMessageBegin() override {} - void onUrl(const char*, size_t) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - int onHeadersComplete() override; + Status onMessageBegin() override { return okStatus(); } + Status onUrl(const char*, size_t) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + Envoy::StatusOr onHeadersComplete() override; bool upgradeAllowed() const override; void onBody(Buffer::Instance& data) override; void onMessageComplete() override; void onResetStream(StreamResetReason reason) override; - void sendProtocolError(absl::string_view details) override; + Status sendProtocolError(absl::string_view details) override; void onAboveHighWatermark() override; void onBelowLowWatermark() override; HeaderMap& headersOrTrailers() override { diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 9c4534a0fdc4..50b4cac3aacf 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -50,7 +50,7 @@ template <> TestRequestHeaderMapImpl fromSanitizedHeaders(const test::fuzz::Headers& headers) { return Fuzz::fromHeaders(headers, {"transfer-encoding"}, - {":authority"}); + {":authority", ":method", ":path"}); } // Convert from test proto Http1ServerSettings to Http1Settings. diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 7ecd8baf1bb0..f6da689eacd6 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -68,8 +68,10 @@ class Http1CodecTestBase { class Http1ServerConnectionImplTest : public Http1CodecTestBase, public testing::TestWithParam { public: + bool testingNewCodec() { return GetParam(); } + void initialize() { - if (GetParam()) { + if (testingNewCodec()) { codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, max_request_headers_count_, headers_with_underscores_action_); @@ -135,7 +137,7 @@ void Http1ServerConnectionImplTest::expect400(Protocol p, bool allow_absolute_ur if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - if (GetParam()) { + if (testingNewCodec()) { codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); @@ -171,7 +173,7 @@ void Http1ServerConnectionImplTest::expectHeadersTest(Protocol p, bool allow_abs // Make a new 'codec' with the right settings if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - if (GetParam()) { + if (testingNewCodec()) { codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); @@ -198,7 +200,7 @@ void Http1ServerConnectionImplTest::expectTrailersTest(bool enable_trailers) { // Make a new 'codec' with the right settings if (enable_trailers) { codec_settings_.enable_trailers_ = enable_trailers; - if (GetParam()) { + if (testingNewCodec()) { codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); @@ -240,7 +242,7 @@ void Http1ServerConnectionImplTest::testTrailersExceedLimit(std::string trailer_ initialize(); // Make a new 'codec' with the right settings codec_settings_.enable_trailers_ = enable_trailers; - if (GetParam()) { + if (testingNewCodec()) { codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_request_headers_kb_, max_request_headers_count_, envoy::config::core::v3::HttpProtocolOptions::ALLOW); @@ -1841,8 +1843,10 @@ TEST_P(Http1ServerConnectionImplTest, WatermarkTest) { class Http1ClientConnectionImplTest : public Http1CodecTestBase, public testing::TestWithParam { public: + bool testingNewCodec() { return GetParam(); } + void initialize() { - if (GetParam()) { + if (testingNewCodec()) { codec_ = std::make_unique( connection_, http1CodecStats(), callbacks_, codec_settings_, max_response_headers_count_); } else { @@ -1852,7 +1856,7 @@ class Http1ClientConnectionImplTest : public Http1CodecTestBase, } void readDisableOnRequestEncoder(RequestEncoder* request_encoder, bool disable) { - if (GetParam()) { + if (testingNewCodec()) { dynamic_cast(request_encoder)->readDisable(disable); } else { dynamic_cast(request_encoder)->readDisable(disable); @@ -2238,12 +2242,23 @@ TEST_P(Http1ClientConnectionImplTest, BadEncodeParams) { NiceMock response_decoder; - // Need to set :method and :path + // Need to set :method and :path. + // New and legacy codecs will behave differently on errors from processing outbound data. The + // legacy codecs will throw an exception (that presently will be uncaught in contexts like + // sendLocalReply), while the new codecs temporarily RELEASE_ASSERT until Envoy handles errors on + // outgoing data. Http::RequestEncoder& request_encoder = codec_->newStream(response_decoder); - EXPECT_THROW(request_encoder.encodeHeaders(TestRequestHeaderMapImpl{{":path", "/"}}, true), - CodecClientException); - EXPECT_THROW(request_encoder.encodeHeaders(TestRequestHeaderMapImpl{{":method", "GET"}}, true), - CodecClientException); + if (testingNewCodec()) { + EXPECT_DEATH(request_encoder.encodeHeaders(TestRequestHeaderMapImpl{{":path", "/"}}, true), + ":method and :path must be specified"); + EXPECT_DEATH(request_encoder.encodeHeaders(TestRequestHeaderMapImpl{{":method", "GET"}}, true), + ":method and :path must be specified"); + } else { + EXPECT_THROW(request_encoder.encodeHeaders(TestRequestHeaderMapImpl{{":path", "/"}}, true), + CodecClientException); + EXPECT_THROW(request_encoder.encodeHeaders(TestRequestHeaderMapImpl{{":method", "GET"}}, true), + CodecClientException); + } } TEST_P(Http1ClientConnectionImplTest, NoContentLengthResponse) { From dc560dfaa0e7ce158eca8a2e8b5ba68246719ad2 Mon Sep 17 00:00:00 2001 From: Martin Matusiak Date: Tue, 11 Aug 2020 01:16:45 +1000 Subject: [PATCH 12/67] cleanup: remove unused forward declaration (#12515) Signed-off-by: Martin Matusiak Commit Message: cleanup: remove unused forward declaration Risk Level: Low Testing: bazel test //test/... Docs Changes: none Release Notes: n/a --- include/envoy/stats/stats.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index c40e072c9d11..c03b1d58ad0b 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -15,8 +15,6 @@ namespace Envoy { namespace Stats { -class Allocator; - /** * General interface for all stats objects. */ From 9ed8092b5eac0874ff028b22fe72d28057ff94a1 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 10 Aug 2020 10:07:45 -0700 Subject: [PATCH 13/67] build: mark virtual functions called in ctor/dtor final (#12558) Signed-off-by: Lizan Zhou Prevents undefined behavior and let clang-tidy not warn about it. --- source/common/network/connection_impl.h | 4 ++-- .../ssl/certificate_validation_context_config_impl.h | 2 +- source/common/stats/allocator_impl.cc | 2 +- source/common/stats/histogram_impl.h | 2 +- source/common/stats/scope_prefixer.h | 4 ++-- source/common/stats/thread_local_store.h | 7 ++++--- source/common/tcp_proxy/tcp_proxy.h | 2 +- source/common/upstream/upstream_impl.h | 2 +- .../common/dynamic_forward_proxy/dns_cache_impl.h | 2 +- source/extensions/filters/http/common/jwks_fetcher.cc | 2 +- source/extensions/filters/http/jwt_authn/jwks_cache.cc | 2 +- source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h | 2 +- 12 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index b464e2af96d1..17ebe609a263 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -60,7 +60,7 @@ class ConnectionImpl : public ConnectionImplBase, public TransportSocketCallback // Network::Connection void addBytesSentCallback(BytesSentCb cb) override; void enableHalfClose(bool enabled) override; - void close(ConnectionCloseType type) override; + void close(ConnectionCloseType type) final; std::string nextProtocol() const override { return transport_socket_->protocol(); } void noDelay(bool enable) override; void readDisable(bool disable) override; @@ -132,7 +132,7 @@ class ConnectionImpl : public ConnectionImplBase, public TransportSocketCallback bool consumerWantsToRead(); // Network::ConnectionImplBase - void closeConnectionImmediately() override; + void closeConnectionImmediately() final; void closeSocket(ConnectionEvent close_type); diff --git a/source/common/ssl/certificate_validation_context_config_impl.h b/source/common/ssl/certificate_validation_context_config_impl.h index f054039ee1ba..1636c2ed0713 100644 --- a/source/common/ssl/certificate_validation_context_config_impl.h +++ b/source/common/ssl/certificate_validation_context_config_impl.h @@ -21,7 +21,7 @@ class CertificateValidationContextConfigImpl : public CertificateValidationConte const std::string& certificateRevocationList() const override { return certificate_revocation_list_; } - const std::string& certificateRevocationListPath() const override { + const std::string& certificateRevocationListPath() const final { return certificate_revocation_list_path_; } const std::vector& verifySubjectAltNameList() const override { diff --git a/source/common/stats/allocator_impl.cc b/source/common/stats/allocator_impl.cc index 5e507db18522..63e3159a842e 100644 --- a/source/common/stats/allocator_impl.cc +++ b/source/common/stats/allocator_impl.cc @@ -63,7 +63,7 @@ template class StatsSharedImpl : public MetricImpl } // Metric - SymbolTable& symbolTable() override { return alloc_.symbolTable(); } + SymbolTable& symbolTable() final { return alloc_.symbolTable(); } bool used() const override { return flags_ & Metric::Flags::Used; } // RefcountInterface diff --git a/source/common/stats/histogram_impl.h b/source/common/stats/histogram_impl.h index a58c60fd5fc5..67c2d7d17066 100644 --- a/source/common/stats/histogram_impl.h +++ b/source/common/stats/histogram_impl.h @@ -108,7 +108,7 @@ class HistogramImpl : public HistogramImplHelper { void recordValue(uint64_t value) override { parent_.deliverHistogramToSinks(*this, value); } bool used() const override { return true; } - SymbolTable& symbolTable() override { return parent_.symbolTable(); } + SymbolTable& symbolTable() final { return parent_.symbolTable(); } private: Unit unit_; diff --git a/source/common/stats/scope_prefixer.h b/source/common/stats/scope_prefixer.h index 4257c1dd5ddf..b6872bc98dff 100644 --- a/source/common/stats/scope_prefixer.h +++ b/source/common/stats/scope_prefixer.h @@ -49,8 +49,8 @@ class ScopePrefixer : public Scope { HistogramOptConstRef findHistogram(StatName name) const override; TextReadoutOptConstRef findTextReadout(StatName name) const override; - const SymbolTable& constSymbolTable() const override { return scope_.constSymbolTable(); } - SymbolTable& symbolTable() override { return scope_.symbolTable(); } + const SymbolTable& constSymbolTable() const final { return scope_.constSymbolTable(); } + SymbolTable& symbolTable() final { return scope_.symbolTable(); } NullGaugeImpl& nullGauge(const std::string& str) override { return scope_.nullGauge(str); } diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 23ce40e5fc15..c86844a2d38c 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -59,7 +59,7 @@ class ThreadLocalHistogramImpl : public HistogramImplHelper { void recordValue(uint64_t value) override; // Stats::Metric - SymbolTable& symbolTable() override { return symbol_table_; } + SymbolTable& symbolTable() final { return symbol_table_; } bool used() const override { return used_; } private: @@ -334,13 +334,14 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo ScopePtr createScope(const std::string& name) override { return parent_.createScope(symbolTable().toString(prefix_.statName()) + "." + name); } - const SymbolTable& constSymbolTable() const override { return parent_.constSymbolTable(); } - SymbolTable& symbolTable() override { return parent_.symbolTable(); } + const SymbolTable& constSymbolTable() const final { return parent_.constSymbolTable(); } + SymbolTable& symbolTable() final { return parent_.symbolTable(); } Counter& counterFromString(const std::string& name) override { StatNameManagedStorage storage(name, symbolTable()); return counterFromStatName(storage.statName()); } + Gauge& gaugeFromString(const std::string& name, Gauge::ImportMode import_mode) override { StatNameManagedStorage storage(name, symbolTable()); return gaugeFromStatName(storage.statName(), import_mode); diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index 871be2ad16f8..8a402e8a4cd2 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -322,7 +322,7 @@ class Filter : public Network::ReadFilter, bool on_high_watermark_called_{false}; }; - virtual StreamInfo::StreamInfo& getStreamInfo(); + StreamInfo::StreamInfo& getStreamInfo(); protected: struct DownstreamCallbacks : public Network::ConnectionCallbacks { diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 999962a5b3b4..4a9e0a06468d 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -196,7 +196,7 @@ class HostImpl : public HostDescriptionImpl, } void healthFlagClear(HealthFlag flag) override { health_flags_ &= ~enumToInt(flag); } bool healthFlagGet(HealthFlag flag) const override { return health_flags_ & enumToInt(flag); } - void healthFlagSet(HealthFlag flag) override { health_flags_ |= enumToInt(flag); } + void healthFlagSet(HealthFlag flag) final { health_flags_ |= enumToInt(flag); } ActiveHealthFailureType getActiveHealthFailureType() const override { return active_health_failure_type_; diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index 6ba35d5a5f31..a7f1426c8be3 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -89,7 +89,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggablecancel(); ENVOY_LOG(debug, "fetch pubkey [uri = {}]: canceled", uri_->uri()); diff --git a/source/extensions/filters/http/jwt_authn/jwks_cache.cc b/source/extensions/filters/http/jwt_authn/jwks_cache.cc index a6020ad9c055..7ec91acd9806 100644 --- a/source/extensions/filters/http/jwt_authn/jwks_cache.cc +++ b/source/extensions/filters/http/jwt_authn/jwks_cache.cc @@ -115,7 +115,7 @@ class JwksCacheImpl : public JwksCache { return it->second; } - JwksData* findByProvider(const std::string& provider) override { + JwksData* findByProvider(const std::string& provider) final { const auto it = jwks_data_map_.find(provider); if (it == jwks_data_map_.end()) { return nullptr; diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h index 68a85b3699d8..90c1f345ac38 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.h @@ -226,7 +226,7 @@ class UdpProxyFilter : public Network::UdpListenerReadFilter, } // Upstream::ClusterUpdateCallbacks - void onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) override; + void onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) final; void onClusterRemoval(const std::string& cluster_name) override; const UdpProxyFilterConfigSharedPtr config_; From 0d74a8bd03e2f54331262b963c158e46a7f8a9fb Mon Sep 17 00:00:00 2001 From: Roelof DuToit Date: Mon, 10 Aug 2020 13:22:27 -0400 Subject: [PATCH 14/67] buffer: add method to extract front slice without copying (#12439) Add a method to Envoy::Buffer::Instance that may be used to extract the front slice of the implementation's queue (SliceDeque in the case of Buffer::OwnedImpl) without copying the actual payload. A SliceData class is defined to facilitate the extraction process. Signed-off-by: Roelof DuToit --- include/envoy/buffer/buffer.h | 25 +++ source/common/buffer/buffer_impl.cc | 28 ++++ source/common/buffer/buffer_impl.h | 45 ++++-- source/common/buffer/watermark_buffer.cc | 15 ++ source/common/buffer/watermark_buffer.h | 3 + test/common/buffer/buffer_fuzz.cc | 2 + test/common/buffer/owned_impl_test.cc | 163 ++++++++++++++++++++ test/common/buffer/watermark_buffer_test.cc | 33 ++++ 8 files changed, 305 insertions(+), 9 deletions(-) diff --git a/include/envoy/buffer/buffer.h b/include/envoy/buffer/buffer.h index aca59b31d695..6e4f52644e37 100644 --- a/include/envoy/buffer/buffer.h +++ b/include/envoy/buffer/buffer.h @@ -16,6 +16,7 @@ #include "absl/container/inlined_vector.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "absl/types/span.h" namespace Envoy { namespace Buffer { @@ -55,6 +56,21 @@ class BufferFragment { virtual void done() PURE; }; +/** + * A class to facilitate extracting buffer slices from a buffer instance. + */ +class SliceData { +public: + virtual ~SliceData() = default; + + /** + * @return a mutable view of the slice data. + */ + virtual absl::Span getMutableData() PURE; +}; + +using SliceDataPtr = std::unique_ptr; + /** * A basic buffer abstraction. */ @@ -144,6 +160,15 @@ class Instance { virtual RawSliceVector getRawSlices(absl::optional max_slices = absl::nullopt) const PURE; + /** + * Transfer ownership of the front slice to the caller. Must only be called if the + * buffer is not empty otherwise the implementation will have undefined behavior. + * If the underlying slice is immutable then the implementation must create and return + * a mutable slice that has a copy of the immutable data. + * @return pointer to SliceData object that wraps the front slice + */ + virtual SliceDataPtr extractMutableFrontSlice() PURE; + /** * @return uint64_t the total length of the buffer (not necessarily contiguous in memory). */ diff --git a/source/common/buffer/buffer_impl.cc b/source/common/buffer/buffer_impl.cc index 0ad095135e57..0b92c7a426f5 100644 --- a/source/common/buffer/buffer_impl.cc +++ b/source/common/buffer/buffer_impl.cc @@ -201,6 +201,34 @@ RawSliceVector OwnedImpl::getRawSlices(absl::optional max_slices) cons return raw_slices; } +SliceDataPtr OwnedImpl::extractMutableFrontSlice() { + RELEASE_ASSERT(length_ > 0, "Extract called on empty buffer"); + // Remove zero byte fragments from the front of the queue to ensure + // that the extracted slice has data. + while (!slices_.empty() && slices_.front()->dataSize() == 0) { + slices_.pop_front(); + } + ASSERT(!slices_.empty()); + ASSERT(slices_.front()); + auto slice = std::move(slices_.front()); + auto size = slice->dataSize(); + length_ -= size; + slices_.pop_front(); + if (!slice->isMutable()) { + // Create a mutable copy of the immutable slice data. + auto mutable_slice = OwnedSlice::create(size); + auto copy_size = mutable_slice->append(slice->data(), size); + ASSERT(copy_size == size); + // Drain trackers for the immutable slice will be called as part of the slice destructor. + return mutable_slice; + } else { + // Make sure drain trackers are called before ownership of the slice is transferred from + // the buffer to the caller. + slice->callAndClearDrainTrackers(); + return slice; + } +} + uint64_t OwnedImpl::length() const { #ifndef NDEBUG // When running in debug mode, verify that the precomputed length matches the sum diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 05e673d6b2ae..92ff88742dc1 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -31,16 +31,23 @@ namespace Buffer { * | * data() */ -class Slice { +class Slice : public SliceData { public: using Reservation = RawSlice; - virtual ~Slice() { - for (const auto& drain_tracker : drain_trackers_) { - drain_tracker(); - } + ~Slice() override { callAndClearDrainTrackers(); } + + // SliceData + absl::Span getMutableData() override { + RELEASE_ASSERT(isMutable(), "Not allowed to call getMutableData if slice is immutable"); + return {base_ + data_, reservable_ - data_}; } + /** + * @return true if the data in the slice is mutable + */ + virtual bool isMutable() const { return false; } + /** * @return a pointer to the start of the usable content. */ @@ -117,10 +124,10 @@ class Slice { * @param reservation a reservation obtained from a previous call to reserve(). * If the reservation is not from this Slice, commit() will return false. * If the caller is committing fewer bytes than provided by reserve(), it - * should change the mem_ field of the reservation before calling commit(). + * should change the len_ field of the reservation before calling commit(). * For example, if a caller reserve()s 4KB to do a nonblocking socket read, * and the read only returns two bytes, the caller should set - * reservation.mem_ = 2 and then call `commit(reservation)`. + * reservation.len_ = 2 and then call `commit(reservation)`. * @return whether the Reservation was successfully committed to the Slice. */ bool commit(const Reservation& reservation) { @@ -200,15 +207,32 @@ class Slice { return SliceRepresentation{dataSize(), reservableSize(), capacity_}; } + /** + * Move all drain trackers from the current slice to the destination slice. + */ void transferDrainTrackersTo(Slice& destination) { destination.drain_trackers_.splice(destination.drain_trackers_.end(), drain_trackers_); ASSERT(drain_trackers_.empty()); } + /** + * Add a drain tracker to the slice. + */ void addDrainTracker(std::function drain_tracker) { drain_trackers_.emplace_back(std::move(drain_tracker)); } + /** + * Call all drain trackers associated with the slice, then clear + * the drain tracker list. + */ + void callAndClearDrainTrackers() { + for (const auto& drain_tracker : drain_trackers_) { + drain_tracker(); + } + drain_trackers_.clear(); + } + protected: Slice(uint64_t data, uint64_t reservable, uint64_t capacity) : data_(data), reservable_(reservable), capacity_(capacity) {} @@ -261,6 +285,8 @@ class OwnedSlice final : public Slice, public InlineStorage { private: OwnedSlice(uint64_t size) : Slice(0, 0, size) { base_ = storage_; } + bool isMutable() const override { return true; } + /** * Compute a slice size big enough to hold a specified amount of data. * @param data_size the minimum amount of data the slice must be able to store, in bytes. @@ -539,6 +565,7 @@ class OwnedImpl : public LibEventInstance { void copyOut(size_t start, uint64_t size, void* data) const override; void drain(uint64_t size) override; RawSliceVector getRawSlices(absl::optional max_slices = absl::nullopt) const override; + SliceDataPtr extractMutableFrontSlice() override; uint64_t length() const override; void* linearize(uint32_t size) override; void move(Instance& rhs) override; @@ -558,13 +585,13 @@ class OwnedImpl : public LibEventInstance { * @param data start of the content to copy. * */ - void appendSliceForTest(const void* data, uint64_t size); + virtual void appendSliceForTest(const void* data, uint64_t size); /** * Create a new slice at the end of the buffer, and copy the supplied string into it. * @param data the string to append to the buffer. */ - void appendSliceForTest(absl::string_view data); + virtual void appendSliceForTest(absl::string_view data); /** * Describe the in-memory representation of the slices in the buffer. For use diff --git a/source/common/buffer/watermark_buffer.cc b/source/common/buffer/watermark_buffer.cc index e3537ffe7943..9d566be1965d 100644 --- a/source/common/buffer/watermark_buffer.cc +++ b/source/common/buffer/watermark_buffer.cc @@ -51,6 +51,12 @@ void WatermarkBuffer::move(Instance& rhs, uint64_t length) { checkHighAndOverflowWatermarks(); } +SliceDataPtr WatermarkBuffer::extractMutableFrontSlice() { + auto result = OwnedImpl::extractMutableFrontSlice(); + checkLowWatermark(); + return result; +} + Api::IoCallUint64Result WatermarkBuffer::read(Network::IoHandle& io_handle, uint64_t max_length) { Api::IoCallUint64Result result = OwnedImpl::read(io_handle, max_length); checkHighAndOverflowWatermarks(); @@ -69,6 +75,15 @@ Api::IoCallUint64Result WatermarkBuffer::write(Network::IoHandle& io_handle) { return result; } +void WatermarkBuffer::appendSliceForTest(const void* data, uint64_t size) { + OwnedImpl::appendSliceForTest(data, size); + checkHighAndOverflowWatermarks(); +} + +void WatermarkBuffer::appendSliceForTest(absl::string_view data) { + appendSliceForTest(data.data(), data.size()); +} + void WatermarkBuffer::setWatermarks(uint32_t low_watermark, uint32_t high_watermark) { ASSERT(low_watermark < high_watermark || (high_watermark == 0 && low_watermark == 0)); uint32_t overflow_watermark_multiplier = diff --git a/source/common/buffer/watermark_buffer.h b/source/common/buffer/watermark_buffer.h index 127069307902..de44822a56ab 100644 --- a/source/common/buffer/watermark_buffer.h +++ b/source/common/buffer/watermark_buffer.h @@ -34,10 +34,13 @@ class WatermarkBuffer : public OwnedImpl { void drain(uint64_t size) override; void move(Instance& rhs) override; void move(Instance& rhs, uint64_t length) override; + SliceDataPtr extractMutableFrontSlice() override; Api::IoCallUint64Result read(Network::IoHandle& io_handle, uint64_t max_length) override; uint64_t reserve(uint64_t length, RawSlice* iovecs, uint64_t num_iovecs) override; Api::IoCallUint64Result write(Network::IoHandle& io_handle) override; void postProcess() override { checkLowWatermark(); } + void appendSliceForTest(const void* data, uint64_t size) override; + void appendSliceForTest(absl::string_view data) override; void setWatermarks(uint32_t watermark) { setWatermarks(watermark / 2, watermark); } void setWatermarks(uint32_t low_watermark, uint32_t high_watermark); diff --git a/test/common/buffer/buffer_fuzz.cc b/test/common/buffer/buffer_fuzz.cc index 9c80f4655b09..5ab1bd85c4ae 100644 --- a/test/common/buffer/buffer_fuzz.cc +++ b/test/common/buffer/buffer_fuzz.cc @@ -133,6 +133,8 @@ class StringBuffer : public Buffer::Instance { return mutableStart(); } + Buffer::SliceDataPtr extractMutableFrontSlice() override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + void move(Buffer::Instance& rhs) override { move(rhs, rhs.length()); } void move(Buffer::Instance& rhs, uint64_t length) override { diff --git a/test/common/buffer/owned_impl_test.cc b/test/common/buffer/owned_impl_test.cc index 42246acb357a..ce7ec99e3847 100644 --- a/test/common/buffer/owned_impl_test.cc +++ b/test/common/buffer/owned_impl_test.cc @@ -347,6 +347,169 @@ TEST_F(OwnedImplTest, Read) { EXPECT_THAT(buffer.describeSlicesForTest(), testing::IsEmpty()); } +TEST_F(OwnedImplTest, ExtractOwnedSlice) { + // Create a buffer with two owned slices. + Buffer::OwnedImpl buffer; + buffer.appendSliceForTest("abcde"); + const uint64_t expected_length0 = 5; + buffer.appendSliceForTest("123"); + const uint64_t expected_length1 = 3; + EXPECT_EQ(buffer.toString(), "abcde123"); + RawSliceVector slices = buffer.getRawSlices(); + EXPECT_EQ(2, slices.size()); + + // Extract first slice. + auto slice = buffer.extractMutableFrontSlice(); + ASSERT_TRUE(slice); + auto slice_data = slice->getMutableData(); + ASSERT_NE(slice_data.data(), nullptr); + EXPECT_EQ(slice_data.size(), expected_length0); + EXPECT_EQ("abcde", + absl::string_view(reinterpret_cast(slice_data.data()), slice_data.size())); + EXPECT_EQ(buffer.toString(), "123"); + + // Modify and re-add extracted first slice data to the end of the buffer. + auto slice_mutable_data = slice->getMutableData(); + ASSERT_NE(slice_mutable_data.data(), nullptr); + EXPECT_EQ(slice_mutable_data.size(), expected_length0); + *slice_mutable_data.data() = 'A'; + buffer.appendSliceForTest(slice_mutable_data.data(), slice_mutable_data.size()); + EXPECT_EQ(buffer.toString(), "123Abcde"); + + // Extract second slice, leaving only the original first slice. + slice = buffer.extractMutableFrontSlice(); + ASSERT_TRUE(slice); + slice_data = slice->getMutableData(); + ASSERT_NE(slice_data.data(), nullptr); + EXPECT_EQ(slice_data.size(), expected_length1); + EXPECT_EQ("123", + absl::string_view(reinterpret_cast(slice_data.data()), slice_data.size())); + EXPECT_EQ(buffer.toString(), "Abcde"); +} + +TEST_F(OwnedImplTest, ExtractAfterSentinelDiscard) { + // Create a buffer with a sentinel and one owned slice. + Buffer::OwnedImpl buffer; + bool sentinel_discarded = false; + const Buffer::OwnedBufferFragmentImpl::Releasor sentinel_releasor{ + [&](const Buffer::OwnedBufferFragmentImpl* sentinel) { + sentinel_discarded = true; + delete sentinel; + }}; + auto sentinel = + Buffer::OwnedBufferFragmentImpl::create(absl::string_view("", 0), sentinel_releasor); + buffer.addBufferFragment(*sentinel.release()); + + buffer.appendSliceForTest("abcde"); + const uint64_t expected_length = 5; + EXPECT_EQ(buffer.toString(), "abcde"); + RawSliceVector slices = buffer.getRawSlices(); // only returns slices with data + EXPECT_EQ(1, slices.size()); + + // Extract owned slice after discarding sentinel. + EXPECT_FALSE(sentinel_discarded); + auto slice = buffer.extractMutableFrontSlice(); + ASSERT_TRUE(slice); + EXPECT_TRUE(sentinel_discarded); + auto slice_data = slice->getMutableData(); + ASSERT_NE(slice_data.data(), nullptr); + EXPECT_EQ(slice_data.size(), expected_length); + EXPECT_EQ("abcde", + absl::string_view(reinterpret_cast(slice_data.data()), slice_data.size())); + EXPECT_EQ(0, buffer.length()); +} + +TEST_F(OwnedImplTest, DrainThenExtractOwnedSlice) { + // Create a buffer with two owned slices. + Buffer::OwnedImpl buffer; + buffer.appendSliceForTest("abcde"); + const uint64_t expected_length0 = 5; + buffer.appendSliceForTest("123"); + EXPECT_EQ(buffer.toString(), "abcde123"); + RawSliceVector slices = buffer.getRawSlices(); + EXPECT_EQ(2, slices.size()); + + // Partially drain the first slice. + const uint64_t partial_drain_size = 2; + buffer.drain(partial_drain_size); + EXPECT_EQ(buffer.toString(), static_cast("abcde123") + partial_drain_size); + + // Extracted partially drained first slice, leaving the second slice. + auto slice = buffer.extractMutableFrontSlice(); + ASSERT_TRUE(slice); + auto slice_data = slice->getMutableData(); + ASSERT_NE(slice_data.data(), nullptr); + EXPECT_EQ(slice_data.size(), expected_length0 - partial_drain_size); + EXPECT_EQ(static_cast("abcde") + partial_drain_size, + absl::string_view(reinterpret_cast(slice_data.data()), slice_data.size())); + EXPECT_EQ(buffer.toString(), "123"); +} + +TEST_F(OwnedImplTest, ExtractUnownedSlice) { + // Create a buffer with an unowned slice. + std::string input{"unowned test slice"}; + const size_t expected_length0 = input.size(); + auto frag = OwnedBufferFragmentImpl::create( + {input.c_str(), expected_length0}, + [this](const OwnedBufferFragmentImpl*) { release_callback_called_ = true; }); + Buffer::OwnedImpl buffer; + buffer.addBufferFragment(*frag); + + bool drain_tracker_called{false}; + buffer.addDrainTracker([&] { drain_tracker_called = true; }); + + // Add an owned slice to the end of the buffer. + EXPECT_EQ(expected_length0, buffer.length()); + std::string owned_slice_content{"another slice, but owned"}; + buffer.add(owned_slice_content); + const uint64_t expected_length1 = owned_slice_content.length(); + + // Partially drain the unowned slice. + const uint64_t partial_drain_size = 5; + buffer.drain(partial_drain_size); + EXPECT_EQ(expected_length0 - partial_drain_size + expected_length1, buffer.length()); + EXPECT_FALSE(release_callback_called_); + EXPECT_FALSE(drain_tracker_called); + + // Extract what remains of the unowned slice, leaving only the owned slice. + auto slice = buffer.extractMutableFrontSlice(); + ASSERT_TRUE(slice); + EXPECT_TRUE(drain_tracker_called); + auto slice_data = slice->getMutableData(); + ASSERT_NE(slice_data.data(), nullptr); + EXPECT_EQ(slice_data.size(), expected_length0 - partial_drain_size); + EXPECT_EQ(input.data() + partial_drain_size, + absl::string_view(reinterpret_cast(slice_data.data()), slice_data.size())); + EXPECT_EQ(expected_length1, buffer.length()); + + // The underlying immutable unowned slice was discarded during the extract + // operation and replaced with a mutable copy. The drain trackers were + // called as part of the extract, implying that the release callback was called. + EXPECT_TRUE(release_callback_called_); +} + +TEST_F(OwnedImplTest, ExtractWithDrainTracker) { + testing::InSequence s; + + Buffer::OwnedImpl buffer; + buffer.add("a"); + + testing::MockFunction tracker1; + testing::MockFunction tracker2; + buffer.addDrainTracker(tracker1.AsStdFunction()); + buffer.addDrainTracker(tracker2.AsStdFunction()); + + testing::MockFunction done; + EXPECT_CALL(tracker1, Call()); + EXPECT_CALL(tracker2, Call()); + EXPECT_CALL(done, Call()); + auto slice = buffer.extractMutableFrontSlice(); + // The test now has ownership of the slice, but the drain trackers were + // called as part of the extract operation + done.Call(); + slice.reset(); +} + TEST_F(OwnedImplTest, DrainTracking) { testing::InSequence s; diff --git a/test/common/buffer/watermark_buffer_test.cc b/test/common/buffer/watermark_buffer_test.cc index 476967254f35..3e7cf0b57eed 100644 --- a/test/common/buffer/watermark_buffer_test.cc +++ b/test/common/buffer/watermark_buffer_test.cc @@ -142,6 +142,7 @@ TEST_F(WatermarkBufferTest, Drain) { buffer_.add(TEN_BYTES, 11); buffer_.drain(5); EXPECT_EQ(6, buffer_.length()); + EXPECT_EQ(1, times_high_watermark_called_); EXPECT_EQ(0, times_low_watermark_called_); // Now drain below. @@ -153,6 +154,38 @@ TEST_F(WatermarkBufferTest, Drain) { EXPECT_EQ(2, times_high_watermark_called_); } +TEST_F(WatermarkBufferTest, DrainUsingExtract) { + // Similar to `Drain` test, but using extractMutableFrontSlice() instead of drain(). + buffer_.add(TEN_BYTES, 10); + ASSERT_EQ(buffer_.length(), 10); + buffer_.extractMutableFrontSlice(); + EXPECT_EQ(0, times_high_watermark_called_); + EXPECT_EQ(0, times_low_watermark_called_); + + // Go above the high watermark then drain down to just at the low watermark. + buffer_.appendSliceForTest(TEN_BYTES, 5); + buffer_.appendSliceForTest(TEN_BYTES, 1); + buffer_.appendSliceForTest(TEN_BYTES, 5); + EXPECT_EQ(1, times_high_watermark_called_); + EXPECT_EQ(0, times_low_watermark_called_); + auto slice0 = buffer_.extractMutableFrontSlice(); // essentially drain(5) + ASSERT_TRUE(slice0); + EXPECT_EQ(slice0->getMutableData().size(), 5); + EXPECT_EQ(6, buffer_.length()); + EXPECT_EQ(0, times_low_watermark_called_); + + // Now drain below. + auto slice1 = buffer_.extractMutableFrontSlice(); // essentially drain(1) + ASSERT_TRUE(slice1); + EXPECT_EQ(slice1->getMutableData().size(), 1); + EXPECT_EQ(1, times_high_watermark_called_); + EXPECT_EQ(1, times_low_watermark_called_); + + // Going back above should trigger the high again. + buffer_.add(TEN_BYTES, 10); + EXPECT_EQ(2, times_high_watermark_called_); +} + // Verify that low watermark callback is called on drain in the case where the // high watermark is non-zero and low watermark is 0. TEST_F(WatermarkBufferTest, DrainWithLowWatermarkOfZero) { From 520389e677cdcd4a85df769deb40f6cdd2f4f6f8 Mon Sep 17 00:00:00 2001 From: Yangmin Zhu Date: Mon, 10 Aug 2020 10:29:44 -0700 Subject: [PATCH 15/67] tap: factor out the TAP filter matcher for later reuse in other filters (#12429) This is the 1st PR for #11832 that factors out the TAP filter matcher to prepare for reuse in other filters. Signed-off-by: Yangmin Zhu --- CODEOWNERS | 2 + api/BUILD | 1 + api/envoy/config/common/matcher/v3/BUILD | 12 ++ .../config/common/matcher/v3/matcher.proto | 100 +++++++++++++++ api/envoy/config/common/matcher/v4alpha/BUILD | 13 ++ .../common/matcher/v4alpha/matcher.proto | 114 ++++++++++++++++++ api/envoy/config/tap/v3/BUILD | 1 + api/envoy/config/tap/v3/common.proto | 13 +- api/envoy/config/tap/v4alpha/BUILD | 1 + api/envoy/config/tap/v4alpha/common.proto | 10 +- api/versioning/BUILD | 1 + docs/root/api-v3/config/common/common.rst | 1 + docs/root/version_history/current.rst | 2 + .../envoy/config/common/matcher/v3/BUILD | 12 ++ .../config/common/matcher/v3/matcher.proto | 100 +++++++++++++++ .../envoy/config/common/matcher/v4alpha/BUILD | 13 ++ .../common/matcher/v4alpha/matcher.proto | 114 ++++++++++++++++++ .../envoy/config/tap/v3/BUILD | 1 + .../envoy/config/tap/v3/common.proto | 13 +- .../envoy/config/tap/v4alpha/BUILD | 1 + .../envoy/config/tap/v4alpha/common.proto | 13 +- source/extensions/common/matcher/BUILD | 21 ++++ .../tap_matcher.cc => matcher/matcher.cc} | 52 ++++---- .../{tap/tap_matcher.h => matcher/matcher.h} | 34 ++---- source/extensions/common/tap/BUILD | 16 +-- source/extensions/common/tap/admin.h | 1 - source/extensions/common/tap/tap.h | 4 +- .../extensions/common/tap/tap_config_base.cc | 21 +++- .../extensions/common/tap/tap_config_base.h | 5 +- test/extensions/common/matcher/BUILD | 19 +++ .../matcher_test.cc} | 24 ++-- test/extensions/common/tap/BUILD | 10 -- test/extensions/common/tap/admin_test.cc | 2 +- test/extensions/filters/http/tap/BUILD | 1 + .../http/tap/tap_filter_integration_test.cc | 99 +++++++++++---- .../filters/http/tap/tap_filter_test.cc | 25 +++- .../tls/integration/ssl_integration_test.cc | 6 +- 37 files changed, 756 insertions(+), 122 deletions(-) create mode 100644 api/envoy/config/common/matcher/v3/BUILD create mode 100644 api/envoy/config/common/matcher/v3/matcher.proto create mode 100644 api/envoy/config/common/matcher/v4alpha/BUILD create mode 100644 api/envoy/config/common/matcher/v4alpha/matcher.proto create mode 100644 generated_api_shadow/envoy/config/common/matcher/v3/BUILD create mode 100644 generated_api_shadow/envoy/config/common/matcher/v3/matcher.proto create mode 100644 generated_api_shadow/envoy/config/common/matcher/v4alpha/BUILD create mode 100644 generated_api_shadow/envoy/config/common/matcher/v4alpha/matcher.proto create mode 100644 source/extensions/common/matcher/BUILD rename source/extensions/common/{tap/tap_matcher.cc => matcher/matcher.cc} (87%) rename source/extensions/common/{tap/tap_matcher.h => matcher/matcher.h} (94%) create mode 100644 test/extensions/common/matcher/BUILD rename test/extensions/common/{tap/tap_matcher_test.cc => matcher/matcher_test.cc} (97%) diff --git a/CODEOWNERS b/CODEOWNERS index 8b206bb0f1c7..3c6ccecfac91 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -80,6 +80,8 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/common/expr @kyessenov @yangminzhu @lizan # webassembly common extension /*/extensions/common/wasm @jplevyak @PiotrSikora @lizan +# common matcher +/*/extensions/common/matcher @mattklein123 @yangminzhu # common crypto extension /*/extensions/common/crypto @lizan @PiotrSikora @bdecoste /*/extensions/common/proxy_protocol @alyssawilk @wez470 diff --git a/api/BUILD b/api/BUILD index 3ac2738ebc3e..99bd1b119c62 100644 --- a/api/BUILD +++ b/api/BUILD @@ -130,6 +130,7 @@ proto_library( "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", "//envoy/config/cluster/v3:pkg", + "//envoy/config/common/matcher/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/config/endpoint/v3:pkg", "//envoy/config/filter/thrift/router/v2alpha1:pkg", diff --git a/api/envoy/config/common/matcher/v3/BUILD b/api/envoy/config/common/matcher/v3/BUILD new file mode 100644 index 000000000000..c312b8eb6a61 --- /dev/null +++ b/api/envoy/config/common/matcher/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/route/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/config/common/matcher/v3/matcher.proto b/api/envoy/config/common/matcher/v3/matcher.proto new file mode 100644 index 000000000000..d0955e7a1f8c --- /dev/null +++ b/api/envoy/config/common/matcher/v3/matcher.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; + +package envoy.config.common.matcher.v3; + +import "envoy/config/route/v3/route_components.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.common.matcher.v3"; +option java_outer_classname = "MatcherProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Unified Matcher API] + +// Match configuration. This is a recursive structure which allows complex nested match +// configurations to be built using various logical operators. +// [#next-free-field: 11] +message MatchPredicate { + // A set of match configurations used for logical operations. + message MatchSet { + // The list of rules that make up the set. + repeated MatchPredicate rules = 1 [(validate.rules).repeated = {min_items: 2}]; + } + + oneof rule { + option (validate.required) = true; + + // A set that describes a logical OR. If any member of the set matches, the match configuration + // matches. + MatchSet or_match = 1; + + // A set that describes a logical AND. If all members of the set match, the match configuration + // matches. + MatchSet and_match = 2; + + // A negation match. The match configuration will match if the negated match condition matches. + MatchPredicate not_match = 3; + + // The match configuration will always match. + bool any_match = 4 [(validate.rules).bool = {const: true}]; + + // HTTP request headers match configuration. + HttpHeadersMatch http_request_headers_match = 5; + + // HTTP request trailers match configuration. + HttpHeadersMatch http_request_trailers_match = 6; + + // HTTP response headers match configuration. + HttpHeadersMatch http_response_headers_match = 7; + + // HTTP response trailers match configuration. + HttpHeadersMatch http_response_trailers_match = 8; + + // HTTP request generic body match configuration. + HttpGenericBodyMatch http_request_generic_body_match = 9; + + // HTTP response generic body match configuration. + HttpGenericBodyMatch http_response_generic_body_match = 10; + } +} + +// HTTP headers match configuration. +message HttpHeadersMatch { + // HTTP headers to match. + repeated route.v3.HeaderMatcher headers = 1; +} + +// HTTP generic body match configuration. +// List of text strings and hex strings to be located in HTTP body. +// All specified strings must be found in the HTTP body for positive match. +// The search may be limited to specified number of bytes from the body start. +// +// .. attention:: +// +// Searching for patterns in HTTP body is potentially cpu intensive. For each specified pattern, http body is scanned byte by byte to find a match. +// If multiple patterns are specified, the process is repeated for each pattern. If location of a pattern is known, ``bytes_limit`` should be specified +// to scan only part of the http body. +message HttpGenericBodyMatch { + message GenericTextMatch { + oneof rule { + option (validate.required) = true; + + // Text string to be located in HTTP body. + string string_match = 1; + + // Sequence of bytes to be located in HTTP body. + bytes binary_match = 2; + } + } + + // Limits search to specified number of bytes - default zero (no limit - match entire captured buffer). + uint32 bytes_limit = 1; + + // List of patterns to match. + repeated GenericTextMatch patterns = 2 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/api/envoy/config/common/matcher/v4alpha/BUILD b/api/envoy/config/common/matcher/v4alpha/BUILD new file mode 100644 index 000000000000..7028ce1a2aea --- /dev/null +++ b/api/envoy/config/common/matcher/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/common/matcher/v3:pkg", + "//envoy/config/route/v4alpha:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/config/common/matcher/v4alpha/matcher.proto b/api/envoy/config/common/matcher/v4alpha/matcher.proto new file mode 100644 index 000000000000..685ae03a1878 --- /dev/null +++ b/api/envoy/config/common/matcher/v4alpha/matcher.proto @@ -0,0 +1,114 @@ +syntax = "proto3"; + +package envoy.config.common.matcher.v4alpha; + +import "envoy/config/route/v4alpha/route_components.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.common.matcher.v4alpha"; +option java_outer_classname = "MatcherProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Unified Matcher API] + +// Match configuration. This is a recursive structure which allows complex nested match +// configurations to be built using various logical operators. +// [#next-free-field: 11] +message MatchPredicate { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.MatchPredicate"; + + // A set of match configurations used for logical operations. + message MatchSet { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.MatchPredicate.MatchSet"; + + // The list of rules that make up the set. + repeated MatchPredicate rules = 1 [(validate.rules).repeated = {min_items: 2}]; + } + + oneof rule { + option (validate.required) = true; + + // A set that describes a logical OR. If any member of the set matches, the match configuration + // matches. + MatchSet or_match = 1; + + // A set that describes a logical AND. If all members of the set match, the match configuration + // matches. + MatchSet and_match = 2; + + // A negation match. The match configuration will match if the negated match condition matches. + MatchPredicate not_match = 3; + + // The match configuration will always match. + bool any_match = 4 [(validate.rules).bool = {const: true}]; + + // HTTP request headers match configuration. + HttpHeadersMatch http_request_headers_match = 5; + + // HTTP request trailers match configuration. + HttpHeadersMatch http_request_trailers_match = 6; + + // HTTP response headers match configuration. + HttpHeadersMatch http_response_headers_match = 7; + + // HTTP response trailers match configuration. + HttpHeadersMatch http_response_trailers_match = 8; + + // HTTP request generic body match configuration. + HttpGenericBodyMatch http_request_generic_body_match = 9; + + // HTTP response generic body match configuration. + HttpGenericBodyMatch http_response_generic_body_match = 10; + } +} + +// HTTP headers match configuration. +message HttpHeadersMatch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.HttpHeadersMatch"; + + // HTTP headers to match. + repeated route.v4alpha.HeaderMatcher headers = 1; +} + +// HTTP generic body match configuration. +// List of text strings and hex strings to be located in HTTP body. +// All specified strings must be found in the HTTP body for positive match. +// The search may be limited to specified number of bytes from the body start. +// +// .. attention:: +// +// Searching for patterns in HTTP body is potentially cpu intensive. For each specified pattern, http body is scanned byte by byte to find a match. +// If multiple patterns are specified, the process is repeated for each pattern. If location of a pattern is known, ``bytes_limit`` should be specified +// to scan only part of the http body. +message HttpGenericBodyMatch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.HttpGenericBodyMatch"; + + message GenericTextMatch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.HttpGenericBodyMatch.GenericTextMatch"; + + oneof rule { + option (validate.required) = true; + + // Text string to be located in HTTP body. + string string_match = 1; + + // Sequence of bytes to be located in HTTP body. + bytes binary_match = 2; + } + } + + // Limits search to specified number of bytes - default zero (no limit - match entire captured buffer). + uint32 bytes_limit = 1; + + // List of patterns to match. + repeated GenericTextMatch patterns = 2 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/api/envoy/config/tap/v3/BUILD b/api/envoy/config/tap/v3/BUILD index f266efc592a2..6fd3142264d9 100644 --- a/api/envoy/config/tap/v3/BUILD +++ b/api/envoy/config/tap/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/common/matcher/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", "//envoy/service/tap/v2alpha:pkg", diff --git a/api/envoy/config/tap/v3/common.proto b/api/envoy/config/tap/v3/common.proto index 81de393e0581..42783115f871 100644 --- a/api/envoy/config/tap/v3/common.proto +++ b/api/envoy/config/tap/v3/common.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.config.tap.v3; +import "envoy/config/common/matcher/v3/matcher.proto"; import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/grpc_service.proto"; import "envoy/config/route/v3/route_components.proto"; @@ -28,7 +29,17 @@ message TapConfig { // The match configuration. If the configuration matches the data source being tapped, a tap will // occur, with the result written to the configured output. - MatchPredicate match_config = 1 [(validate.rules).message = {required: true}]; + // Exactly one of :ref:`match ` and + // :ref:`match_config ` must be set. If both + // are set, the :ref:`match ` will be used. + MatchPredicate match_config = 1 [deprecated = true]; + + // The match configuration. If the configuration matches the data source being tapped, a tap will + // occur, with the result written to the configured output. + // Exactly one of :ref:`match ` and + // :ref:`match_config ` must be set. If both + // are set, the :ref:`match ` will be used. + common.matcher.v3.MatchPredicate match = 4; // The tap output configuration. If a match configuration matches a data source being tapped, // a tap will occur and the data will be written to the configured output. diff --git a/api/envoy/config/tap/v4alpha/BUILD b/api/envoy/config/tap/v4alpha/BUILD index cb06389f0186..be8b1c3a17e3 100644 --- a/api/envoy/config/tap/v4alpha/BUILD +++ b/api/envoy/config/tap/v4alpha/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/common/matcher/v4alpha:pkg", "//envoy/config/core/v4alpha:pkg", "//envoy/config/route/v4alpha:pkg", "//envoy/config/tap/v3:pkg", diff --git a/api/envoy/config/tap/v4alpha/common.proto b/api/envoy/config/tap/v4alpha/common.proto index 5ce87d5b5770..8366187fd1bf 100644 --- a/api/envoy/config/tap/v4alpha/common.proto +++ b/api/envoy/config/tap/v4alpha/common.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.config.tap.v4alpha; +import "envoy/config/common/matcher/v4alpha/matcher.proto"; import "envoy/config/core/v4alpha/base.proto"; import "envoy/config/core/v4alpha/grpc_service.proto"; import "envoy/config/route/v4alpha/route_components.proto"; @@ -25,9 +26,16 @@ message TapConfig { option (udpa.annotations.versioning).previous_message_type = "envoy.config.tap.v3.TapConfig"; + reserved 1; + + reserved "match_config"; + // The match configuration. If the configuration matches the data source being tapped, a tap will // occur, with the result written to the configured output. - MatchPredicate match_config = 1 [(validate.rules).message = {required: true}]; + // Exactly one of :ref:`match ` and + // :ref:`match_config ` must be set. If both + // are set, the :ref:`match ` will be used. + common.matcher.v4alpha.MatchPredicate match = 4; // The tap output configuration. If a match configuration matches a data source being tapped, // a tap will occur and the data will be written to the configured output. diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 950594d7213e..e0a67d2f3cb1 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -13,6 +13,7 @@ proto_library( "//envoy/config/accesslog/v3:pkg", "//envoy/config/bootstrap/v3:pkg", "//envoy/config/cluster/v3:pkg", + "//envoy/config/common/matcher/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/config/endpoint/v3:pkg", "//envoy/config/filter/thrift/router/v2alpha1:pkg", diff --git a/docs/root/api-v3/config/common/common.rst b/docs/root/api-v3/config/common/common.rst index 5739dffe3676..bb6965a5f149 100644 --- a/docs/root/api-v3/config/common/common.rst +++ b/docs/root/api-v3/config/common/common.rst @@ -5,5 +5,6 @@ Common :glob: :maxdepth: 2 + matcher/v3/* ../../extensions/common/dynamic_forward_proxy/v3/* ../../extensions/common/tap/v3/* diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 01a2b7235bee..f5b6217a1757 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -76,3 +76,5 @@ Deprecated ---------- * The :ref:`track_timeout_budgets ` field has been deprecated in favor of `timeout_budgets` part of an :ref:`Optional Configuration `. +* tap: the :ref:`match_config ` field has been deprecated in favor of + :ref:`match ` field. diff --git a/generated_api_shadow/envoy/config/common/matcher/v3/BUILD b/generated_api_shadow/envoy/config/common/matcher/v3/BUILD new file mode 100644 index 000000000000..c312b8eb6a61 --- /dev/null +++ b/generated_api_shadow/envoy/config/common/matcher/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/route/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/config/common/matcher/v3/matcher.proto b/generated_api_shadow/envoy/config/common/matcher/v3/matcher.proto new file mode 100644 index 000000000000..d0955e7a1f8c --- /dev/null +++ b/generated_api_shadow/envoy/config/common/matcher/v3/matcher.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; + +package envoy.config.common.matcher.v3; + +import "envoy/config/route/v3/route_components.proto"; + +import "udpa/annotations/migrate.proto"; +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.common.matcher.v3"; +option java_outer_classname = "MatcherProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Unified Matcher API] + +// Match configuration. This is a recursive structure which allows complex nested match +// configurations to be built using various logical operators. +// [#next-free-field: 11] +message MatchPredicate { + // A set of match configurations used for logical operations. + message MatchSet { + // The list of rules that make up the set. + repeated MatchPredicate rules = 1 [(validate.rules).repeated = {min_items: 2}]; + } + + oneof rule { + option (validate.required) = true; + + // A set that describes a logical OR. If any member of the set matches, the match configuration + // matches. + MatchSet or_match = 1; + + // A set that describes a logical AND. If all members of the set match, the match configuration + // matches. + MatchSet and_match = 2; + + // A negation match. The match configuration will match if the negated match condition matches. + MatchPredicate not_match = 3; + + // The match configuration will always match. + bool any_match = 4 [(validate.rules).bool = {const: true}]; + + // HTTP request headers match configuration. + HttpHeadersMatch http_request_headers_match = 5; + + // HTTP request trailers match configuration. + HttpHeadersMatch http_request_trailers_match = 6; + + // HTTP response headers match configuration. + HttpHeadersMatch http_response_headers_match = 7; + + // HTTP response trailers match configuration. + HttpHeadersMatch http_response_trailers_match = 8; + + // HTTP request generic body match configuration. + HttpGenericBodyMatch http_request_generic_body_match = 9; + + // HTTP response generic body match configuration. + HttpGenericBodyMatch http_response_generic_body_match = 10; + } +} + +// HTTP headers match configuration. +message HttpHeadersMatch { + // HTTP headers to match. + repeated route.v3.HeaderMatcher headers = 1; +} + +// HTTP generic body match configuration. +// List of text strings and hex strings to be located in HTTP body. +// All specified strings must be found in the HTTP body for positive match. +// The search may be limited to specified number of bytes from the body start. +// +// .. attention:: +// +// Searching for patterns in HTTP body is potentially cpu intensive. For each specified pattern, http body is scanned byte by byte to find a match. +// If multiple patterns are specified, the process is repeated for each pattern. If location of a pattern is known, ``bytes_limit`` should be specified +// to scan only part of the http body. +message HttpGenericBodyMatch { + message GenericTextMatch { + oneof rule { + option (validate.required) = true; + + // Text string to be located in HTTP body. + string string_match = 1; + + // Sequence of bytes to be located in HTTP body. + bytes binary_match = 2; + } + } + + // Limits search to specified number of bytes - default zero (no limit - match entire captured buffer). + uint32 bytes_limit = 1; + + // List of patterns to match. + repeated GenericTextMatch patterns = 2 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/generated_api_shadow/envoy/config/common/matcher/v4alpha/BUILD b/generated_api_shadow/envoy/config/common/matcher/v4alpha/BUILD new file mode 100644 index 000000000000..7028ce1a2aea --- /dev/null +++ b/generated_api_shadow/envoy/config/common/matcher/v4alpha/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/common/matcher/v3:pkg", + "//envoy/config/route/v4alpha:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/generated_api_shadow/envoy/config/common/matcher/v4alpha/matcher.proto b/generated_api_shadow/envoy/config/common/matcher/v4alpha/matcher.proto new file mode 100644 index 000000000000..685ae03a1878 --- /dev/null +++ b/generated_api_shadow/envoy/config/common/matcher/v4alpha/matcher.proto @@ -0,0 +1,114 @@ +syntax = "proto3"; + +package envoy.config.common.matcher.v4alpha; + +import "envoy/config/route/v4alpha/route_components.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.config.common.matcher.v4alpha"; +option java_outer_classname = "MatcherProto"; +option java_multiple_files = true; +option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSION_CANDIDATE; + +// [#protodoc-title: Unified Matcher API] + +// Match configuration. This is a recursive structure which allows complex nested match +// configurations to be built using various logical operators. +// [#next-free-field: 11] +message MatchPredicate { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.MatchPredicate"; + + // A set of match configurations used for logical operations. + message MatchSet { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.MatchPredicate.MatchSet"; + + // The list of rules that make up the set. + repeated MatchPredicate rules = 1 [(validate.rules).repeated = {min_items: 2}]; + } + + oneof rule { + option (validate.required) = true; + + // A set that describes a logical OR. If any member of the set matches, the match configuration + // matches. + MatchSet or_match = 1; + + // A set that describes a logical AND. If all members of the set match, the match configuration + // matches. + MatchSet and_match = 2; + + // A negation match. The match configuration will match if the negated match condition matches. + MatchPredicate not_match = 3; + + // The match configuration will always match. + bool any_match = 4 [(validate.rules).bool = {const: true}]; + + // HTTP request headers match configuration. + HttpHeadersMatch http_request_headers_match = 5; + + // HTTP request trailers match configuration. + HttpHeadersMatch http_request_trailers_match = 6; + + // HTTP response headers match configuration. + HttpHeadersMatch http_response_headers_match = 7; + + // HTTP response trailers match configuration. + HttpHeadersMatch http_response_trailers_match = 8; + + // HTTP request generic body match configuration. + HttpGenericBodyMatch http_request_generic_body_match = 9; + + // HTTP response generic body match configuration. + HttpGenericBodyMatch http_response_generic_body_match = 10; + } +} + +// HTTP headers match configuration. +message HttpHeadersMatch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.HttpHeadersMatch"; + + // HTTP headers to match. + repeated route.v4alpha.HeaderMatcher headers = 1; +} + +// HTTP generic body match configuration. +// List of text strings and hex strings to be located in HTTP body. +// All specified strings must be found in the HTTP body for positive match. +// The search may be limited to specified number of bytes from the body start. +// +// .. attention:: +// +// Searching for patterns in HTTP body is potentially cpu intensive. For each specified pattern, http body is scanned byte by byte to find a match. +// If multiple patterns are specified, the process is repeated for each pattern. If location of a pattern is known, ``bytes_limit`` should be specified +// to scan only part of the http body. +message HttpGenericBodyMatch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.HttpGenericBodyMatch"; + + message GenericTextMatch { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.common.matcher.v3.HttpGenericBodyMatch.GenericTextMatch"; + + oneof rule { + option (validate.required) = true; + + // Text string to be located in HTTP body. + string string_match = 1; + + // Sequence of bytes to be located in HTTP body. + bytes binary_match = 2; + } + } + + // Limits search to specified number of bytes - default zero (no limit - match entire captured buffer). + uint32 bytes_limit = 1; + + // List of patterns to match. + repeated GenericTextMatch patterns = 2 [(validate.rules).repeated = {min_items: 1}]; +} diff --git a/generated_api_shadow/envoy/config/tap/v3/BUILD b/generated_api_shadow/envoy/config/tap/v3/BUILD index f266efc592a2..6fd3142264d9 100644 --- a/generated_api_shadow/envoy/config/tap/v3/BUILD +++ b/generated_api_shadow/envoy/config/tap/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/common/matcher/v3:pkg", "//envoy/config/core/v3:pkg", "//envoy/config/route/v3:pkg", "//envoy/service/tap/v2alpha:pkg", diff --git a/generated_api_shadow/envoy/config/tap/v3/common.proto b/generated_api_shadow/envoy/config/tap/v3/common.proto index 81de393e0581..42783115f871 100644 --- a/generated_api_shadow/envoy/config/tap/v3/common.proto +++ b/generated_api_shadow/envoy/config/tap/v3/common.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.config.tap.v3; +import "envoy/config/common/matcher/v3/matcher.proto"; import "envoy/config/core/v3/base.proto"; import "envoy/config/core/v3/grpc_service.proto"; import "envoy/config/route/v3/route_components.proto"; @@ -28,7 +29,17 @@ message TapConfig { // The match configuration. If the configuration matches the data source being tapped, a tap will // occur, with the result written to the configured output. - MatchPredicate match_config = 1 [(validate.rules).message = {required: true}]; + // Exactly one of :ref:`match ` and + // :ref:`match_config ` must be set. If both + // are set, the :ref:`match ` will be used. + MatchPredicate match_config = 1 [deprecated = true]; + + // The match configuration. If the configuration matches the data source being tapped, a tap will + // occur, with the result written to the configured output. + // Exactly one of :ref:`match ` and + // :ref:`match_config ` must be set. If both + // are set, the :ref:`match ` will be used. + common.matcher.v3.MatchPredicate match = 4; // The tap output configuration. If a match configuration matches a data source being tapped, // a tap will occur and the data will be written to the configured output. diff --git a/generated_api_shadow/envoy/config/tap/v4alpha/BUILD b/generated_api_shadow/envoy/config/tap/v4alpha/BUILD index cb06389f0186..be8b1c3a17e3 100644 --- a/generated_api_shadow/envoy/config/tap/v4alpha/BUILD +++ b/generated_api_shadow/envoy/config/tap/v4alpha/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/common/matcher/v4alpha:pkg", "//envoy/config/core/v4alpha:pkg", "//envoy/config/route/v4alpha:pkg", "//envoy/config/tap/v3:pkg", diff --git a/generated_api_shadow/envoy/config/tap/v4alpha/common.proto b/generated_api_shadow/envoy/config/tap/v4alpha/common.proto index 5ce87d5b5770..d18ba1db94c1 100644 --- a/generated_api_shadow/envoy/config/tap/v4alpha/common.proto +++ b/generated_api_shadow/envoy/config/tap/v4alpha/common.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.config.tap.v4alpha; +import "envoy/config/common/matcher/v4alpha/matcher.proto"; import "envoy/config/core/v4alpha/base.proto"; import "envoy/config/core/v4alpha/grpc_service.proto"; import "envoy/config/route/v4alpha/route_components.proto"; @@ -27,7 +28,17 @@ message TapConfig { // The match configuration. If the configuration matches the data source being tapped, a tap will // occur, with the result written to the configured output. - MatchPredicate match_config = 1 [(validate.rules).message = {required: true}]; + // Exactly one of :ref:`match ` and + // :ref:`match_config ` must be set. If both + // are set, the :ref:`match ` will be used. + MatchPredicate hidden_envoy_deprecated_match_config = 1 [deprecated = true]; + + // The match configuration. If the configuration matches the data source being tapped, a tap will + // occur, with the result written to the configured output. + // Exactly one of :ref:`match ` and + // :ref:`match_config ` must be set. If both + // are set, the :ref:`match ` will be used. + common.matcher.v4alpha.MatchPredicate match = 4; // The tap output configuration. If a match configuration matches a data source being tapped, // a tap will occur and the data will be written to the configured output. diff --git a/source/extensions/common/matcher/BUILD b/source/extensions/common/matcher/BUILD new file mode 100644 index 000000000000..2ad3f963048a --- /dev/null +++ b/source/extensions/common/matcher/BUILD @@ -0,0 +1,21 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "matcher_lib", + srcs = ["matcher.cc"], + hdrs = ["matcher.h"], + deps = [ + "//source/common/buffer:buffer_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/http:header_utility_lib", + "@envoy_api//envoy/config/common/matcher/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/common/tap/tap_matcher.cc b/source/extensions/common/matcher/matcher.cc similarity index 87% rename from source/extensions/common/tap/tap_matcher.cc rename to source/extensions/common/matcher/matcher.cc index 71c270432563..8040b4650bca 100644 --- a/source/extensions/common/tap/tap_matcher.cc +++ b/source/extensions/common/matcher/matcher.cc @@ -1,60 +1,58 @@ -#include "extensions/common/tap/tap_matcher.h" - -#include "envoy/config/tap/v3/common.pb.h" +#include "extensions/common/matcher/matcher.h" #include "common/common/assert.h" namespace Envoy { namespace Extensions { namespace Common { -namespace Tap { +namespace Matcher { -void buildMatcher(const envoy::config::tap::v3::MatchPredicate& match_config, +void buildMatcher(const envoy::config::common::matcher::v3::MatchPredicate& match_config, std::vector& matchers) { // In order to store indexes and build our matcher tree inline, we must reserve a slot where // the matcher we are about to create will go. This allows us to know its future index and still // construct more of the tree in each called constructor (e.g., multiple OR/AND conditions). - // Once fully constructed, we move the matcher into its position below. See the tap matcher - // overview in tap.h for more information. + // Once fully constructed, we move the matcher into its position below. See the matcher + // overview in matcher.h for more information. matchers.emplace_back(nullptr); MatcherPtr new_matcher; switch (match_config.rule_case()) { - case envoy::config::tap::v3::MatchPredicate::RuleCase::kOrMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kOrMatch: new_matcher = std::make_unique(match_config.or_match(), matchers, SetLogicMatcher::Type::Or); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kAndMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kAndMatch: new_matcher = std::make_unique(match_config.and_match(), matchers, SetLogicMatcher::Type::And); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kNotMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kNotMatch: new_matcher = std::make_unique(match_config.not_match(), matchers); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kAnyMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kAnyMatch: new_matcher = std::make_unique(matchers); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kHttpRequestHeadersMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kHttpRequestHeadersMatch: new_matcher = std::make_unique( match_config.http_request_headers_match(), matchers); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kHttpRequestTrailersMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kHttpRequestTrailersMatch: new_matcher = std::make_unique( match_config.http_request_trailers_match(), matchers); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kHttpResponseHeadersMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kHttpResponseHeadersMatch: new_matcher = std::make_unique( match_config.http_response_headers_match(), matchers); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kHttpResponseTrailersMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kHttpResponseTrailersMatch: new_matcher = std::make_unique( match_config.http_response_trailers_match(), matchers); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kHttpRequestGenericBodyMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kHttpRequestGenericBodyMatch: new_matcher = std::make_unique( match_config.http_request_generic_body_match(), matchers); break; - case envoy::config::tap::v3::MatchPredicate::RuleCase::kHttpResponseGenericBodyMatch: + case envoy::config::common::matcher::v3::MatchPredicate::RuleCase::kHttpResponseGenericBodyMatch: new_matcher = std::make_unique( match_config.http_response_generic_body_match(), matchers); break; @@ -66,8 +64,9 @@ void buildMatcher(const envoy::config::tap::v3::MatchPredicate& match_config, matchers[new_matcher->index()] = std::move(new_matcher); } -SetLogicMatcher::SetLogicMatcher(const envoy::config::tap::v3::MatchPredicate::MatchSet& configs, - std::vector& matchers, Type type) +SetLogicMatcher::SetLogicMatcher( + const envoy::config::common::matcher::v3::MatchPredicate::MatchSet& configs, + std::vector& matchers, Type type) : LogicMatcherBase(matchers), matchers_(matchers), type_(type) { for (const auto& config : configs.rules()) { indexes_.push_back(matchers_.size()); @@ -100,7 +99,7 @@ void SetLogicMatcher::updateLocalStatus(MatchStatusVector& statuses, [&statuses](size_t index) { return statuses[index].might_change_status_; }); } -NotMatcher::NotMatcher(const envoy::config::tap::v3::MatchPredicate& config, +NotMatcher::NotMatcher(const envoy::config::common::matcher::v3::MatchPredicate& config, std::vector& matchers) : LogicMatcherBase(matchers), matchers_(matchers), not_index_(matchers.size()) { buildMatcher(config, matchers); @@ -117,8 +116,9 @@ void NotMatcher::updateLocalStatus(MatchStatusVector& statuses, statuses[my_index_].might_change_status_ = statuses[not_index_].might_change_status_; } -HttpHeaderMatcherBase::HttpHeaderMatcherBase(const envoy::config::tap::v3::HttpHeadersMatch& config, - const std::vector& matchers) +HttpHeaderMatcherBase::HttpHeaderMatcherBase( + const envoy::config::common::matcher::v3::HttpHeadersMatch& config, + const std::vector& matchers) : SimpleMatcher(matchers), headers_to_match_(Http::HeaderUtility::buildHeaderDataVector(config.headers())) {} @@ -134,18 +134,18 @@ void HttpHeaderMatcherBase::matchHeaders(const Http::HeaderMap& headers, // HTTP body may be passed to the matcher in chunks. The search logic buffers // only as many bytes as is the length of the longest pattern to be found. HttpGenericBodyMatcher::HttpGenericBodyMatcher( - const envoy::config::tap::v3::HttpGenericBodyMatch& config, + const envoy::config::common::matcher::v3::HttpGenericBodyMatch& config, const std::vector& matchers) : HttpBodyMatcherBase(matchers) { patterns_ = std::make_shared>(); for (const auto& i : config.patterns()) { switch (i.rule_case()) { // For binary match 'i' contains sequence of bytes to locate in the body. - case envoy::config::tap::v3::HttpGenericBodyMatch::GenericTextMatch::kBinaryMatch: { + case envoy::config::common::matcher::v3::HttpGenericBodyMatch::GenericTextMatch::kBinaryMatch: { patterns_->push_back(i.binary_match()); } break; // For string match 'i' contains exact string to locate in the body. - case envoy::config::tap::v3::HttpGenericBodyMatch::GenericTextMatch::kStringMatch: + case envoy::config::common::matcher::v3::HttpGenericBodyMatch::GenericTextMatch::kStringMatch: patterns_->push_back(i.string_match()); break; default: @@ -329,7 +329,7 @@ void HttpGenericBodyMatcher::resizeOverlapBuffer(HttpGenericBodyMatcherCtx* ctx) } } -} // namespace Tap +} // namespace Matcher } // namespace Common } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/common/tap/tap_matcher.h b/source/extensions/common/matcher/matcher.h similarity index 94% rename from source/extensions/common/tap/tap_matcher.h rename to source/extensions/common/matcher/matcher.h index 79705e3fe924..4eecd25d3786 100644 --- a/source/extensions/common/tap/tap_matcher.h +++ b/source/extensions/common/matcher/matcher.h @@ -1,15 +1,14 @@ #pragma once -#include "envoy/config/tap/v3/common.pb.h" +#include "envoy/config/common/matcher/v3/matcher.pb.h" #include "common/buffer/buffer_impl.h" -#include "common/common/matchers.h" #include "common/http/header_utility.h" namespace Envoy { namespace Extensions { namespace Common { -namespace Tap { +namespace Matcher { class Matcher; using MatcherPtr = std::unique_ptr; @@ -27,9 +26,9 @@ class MatcherCtx { }; /** - * Base class for all tap matchers. + * Base class for all matchers. * - * A high level note on the design of tap matching which is different from other matching in Envoy + * A high level note on the design of matching which is different from other matching in Envoy * due to a requirement to support streaming matching (match as new data arrives versus * calculating the match given all available data at once). * - The matching system is composed of a constant matching configuration. This is essentially @@ -66,7 +65,7 @@ class Matcher { Matcher(const std::vector& matchers) // NOTE: This code assumes that the index for the matcher being constructed has already been // allocated, which is why my_index_ is set to size() - 1. See buildMatcher() in - // tap_matcher.cc. + // matcher.cc. : my_index_(matchers.size() - 1) {} virtual ~Matcher() = default; @@ -150,9 +149,9 @@ class Matcher { /** * Factory method to build a matcher given a match config. Calling this function may end * up recursively building many matchers, which will all be added to the passed in vector - * of matchers. See the comments in tap.h for the general structure of how tap matchers work. + * of matchers. See the comments in matcher.h for the general structure of how matchers work. */ -void buildMatcher(const envoy::config::tap::v3::MatchPredicate& match_config, +void buildMatcher(const envoy::config::common::matcher::v3::MatchPredicate& match_config, std::vector& matchers); /** @@ -162,7 +161,6 @@ class LogicMatcherBase : public Matcher { public: using Matcher::Matcher; - // Extensions::Common::Tap::Matcher void onNewStream(MatchStatusVector& statuses) const override { updateLocalStatus(statuses, [](Matcher& m, MatchStatusVector& statuses) { m.onNewStream(statuses); }); @@ -215,7 +213,7 @@ class SetLogicMatcher : public LogicMatcherBase { public: enum class Type { And, Or }; - SetLogicMatcher(const envoy::config::tap::v3::MatchPredicate::MatchSet& configs, + SetLogicMatcher(const envoy::config::common::matcher::v3::MatchPredicate::MatchSet& configs, std::vector& matchers, Type type); private: @@ -231,7 +229,7 @@ class SetLogicMatcher : public LogicMatcherBase { */ class NotMatcher : public LogicMatcherBase { public: - NotMatcher(const envoy::config::tap::v3::MatchPredicate& config, + NotMatcher(const envoy::config::common::matcher::v3::MatchPredicate& config, std::vector& matchers); private: @@ -249,7 +247,6 @@ class SimpleMatcher : public Matcher { public: using Matcher::Matcher; - // Extensions::Common::Tap::Matcher void onNewStream(MatchStatusVector&) const override {} void onHttpRequestHeaders(const Http::RequestHeaderMap&, MatchStatusVector&) const override {} void onHttpRequestTrailers(const Http::RequestTrailerMap&, MatchStatusVector&) const override {} @@ -266,7 +263,6 @@ class AnyMatcher : public SimpleMatcher { public: using SimpleMatcher::SimpleMatcher; - // Extensions::Common::Tap::Matcher void onNewStream(MatchStatusVector& statuses) const override { statuses[my_index_].matches_ = true; statuses[my_index_].might_change_status_ = false; @@ -278,7 +274,7 @@ class AnyMatcher : public SimpleMatcher { */ class HttpHeaderMatcherBase : public SimpleMatcher { public: - HttpHeaderMatcherBase(const envoy::config::tap::v3::HttpHeadersMatch& config, + HttpHeaderMatcherBase(const envoy::config::common::matcher::v3::HttpHeadersMatch& config, const std::vector& matchers); protected: @@ -294,7 +290,6 @@ class HttpRequestHeadersMatcher : public HttpHeaderMatcherBase { public: using HttpHeaderMatcherBase::HttpHeaderMatcherBase; - // Extensions::Common::Tap::Matcher void onHttpRequestHeaders(const Http::RequestHeaderMap& request_headers, MatchStatusVector& statuses) const override { matchHeaders(request_headers, statuses); @@ -308,7 +303,6 @@ class HttpRequestTrailersMatcher : public HttpHeaderMatcherBase { public: using HttpHeaderMatcherBase::HttpHeaderMatcherBase; - // Extensions::Common::Tap::Matcher void onHttpRequestTrailers(const Http::RequestTrailerMap& request_trailers, MatchStatusVector& statuses) const override { matchHeaders(request_trailers, statuses); @@ -322,7 +316,6 @@ class HttpResponseHeadersMatcher : public HttpHeaderMatcherBase { public: using HttpHeaderMatcherBase::HttpHeaderMatcherBase; - // Extensions::Common::Tap::Matcher void onHttpResponseHeaders(const Http::ResponseHeaderMap& response_headers, MatchStatusVector& statuses) const override { matchHeaders(response_headers, statuses); @@ -336,7 +329,6 @@ class HttpResponseTrailersMatcher : public HttpHeaderMatcherBase { public: using HttpHeaderMatcherBase::HttpHeaderMatcherBase; - // Extensions::Common::Tap::Matcher void onHttpResponseTrailers(const Http::ResponseTrailerMap& response_trailers, MatchStatusVector& statuses) const override { matchHeaders(response_trailers, statuses); @@ -404,7 +396,7 @@ class HttpGenericBodyMatcherCtx : public MatcherCtx { class HttpGenericBodyMatcher : public HttpBodyMatcherBase { public: - HttpGenericBodyMatcher(const envoy::config::tap::v3::HttpGenericBodyMatch& config, + HttpGenericBodyMatcher(const envoy::config::common::matcher::v3::HttpGenericBodyMatch& config, const std::vector& matchers); protected: @@ -425,7 +417,7 @@ class HttpGenericBodyMatcher : public HttpBodyMatcherBase { private: // The following fields are initialized based on matcher config and are used - // by all HTTP tappers. + // by all HTTP matchers. // List of strings which body must contain to get match. std::shared_ptr> patterns_; // Stores the length of the longest pattern. @@ -450,7 +442,7 @@ class HttpResponseGenericBodyMatcher : public HttpGenericBodyMatcher { } }; -} // namespace Tap +} // namespace Matcher } // namespace Common } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/common/tap/BUILD b/source/extensions/common/tap/BUILD index 8cf381c67dee..e127bf3aaa19 100644 --- a/source/extensions/common/tap/BUILD +++ b/source/extensions/common/tap/BUILD @@ -12,8 +12,8 @@ envoy_cc_library( name = "tap_interface", hdrs = ["tap.h"], deps = [ - ":tap_matcher", "//include/envoy/http:header_map_interface", + "//source/extensions/common/matcher:matcher_lib", "@envoy_api//envoy/config/tap/v3:pkg_cc_proto", "@envoy_api//envoy/data/tap/v3:pkg_cc_proto", ], @@ -25,26 +25,14 @@ envoy_cc_library( hdrs = ["tap_config_base.h"], deps = [ ":tap_interface", - ":tap_matcher", "//source/common/common:assert_lib", "//source/common/common:hex_lib", + "//source/extensions/common/matcher:matcher_lib", "@envoy_api//envoy/config/tap/v3:pkg_cc_proto", "@envoy_api//envoy/data/tap/v3:pkg_cc_proto", ], ) -envoy_cc_library( - name = "tap_matcher", - srcs = ["tap_matcher.cc"], - hdrs = ["tap_matcher.h"], - deps = [ - "//source/common/buffer:buffer_lib", - "//source/common/common:matchers_lib", - "//source/common/http:header_utility_lib", - "@envoy_api//envoy/config/tap/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "admin", srcs = ["admin.cc"], diff --git a/source/extensions/common/tap/admin.h b/source/extensions/common/tap/admin.h index bf80f6889b17..a3cbb7f6e815 100644 --- a/source/extensions/common/tap/admin.h +++ b/source/extensions/common/tap/admin.h @@ -1,6 +1,5 @@ #pragma once -#include "envoy/config/tap/v3/common.pb.h" #include "envoy/server/admin.h" #include "envoy/singleton/manager.h" diff --git a/source/extensions/common/tap/tap.h b/source/extensions/common/tap/tap.h index 58ba4ba82d6d..9abf88d6965b 100644 --- a/source/extensions/common/tap/tap.h +++ b/source/extensions/common/tap/tap.h @@ -5,7 +5,7 @@ #include "envoy/data/tap/v3/wrapper.pb.h" #include "envoy/http/header_map.h" -#include "extensions/common/tap/tap_matcher.h" +#include "extensions/common/matcher/matcher.h" #include "absl/strings/string_view.h" @@ -14,6 +14,8 @@ namespace Extensions { namespace Common { namespace Tap { +using Matcher = Envoy::Extensions::Common::Matcher::Matcher; + using TraceWrapperPtr = std::unique_ptr; inline TraceWrapperPtr makeTraceWrapper() { return std::make_unique(); diff --git a/source/extensions/common/tap/tap_config_base.cc b/source/extensions/common/tap/tap_config_base.cc index b9debd1720e6..7eea4b7fe7be 100644 --- a/source/extensions/common/tap/tap_config_base.cc +++ b/source/extensions/common/tap/tap_config_base.cc @@ -5,9 +5,11 @@ #include "envoy/data/tap/v3/wrapper.pb.h" #include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/config/version_converter.h" #include "common/protobuf/utility.h" -#include "extensions/common/tap/tap_matcher.h" +#include "extensions/common/matcher/matcher.h" #include "absl/container/fixed_array.h" @@ -16,6 +18,8 @@ namespace Extensions { namespace Common { namespace Tap { +using namespace Matcher; + bool Utility::addBufferToProtoBytes(envoy::data::tap::v3::Body& output_body, uint32_t max_buffered_bytes, const Buffer::Instance& data, uint32_t buffer_start_offset, uint32_t buffer_length_to_copy) { @@ -72,7 +76,20 @@ TapConfigBaseImpl::TapConfigBaseImpl(envoy::config::tap::v3::TapConfig&& proto_c NOT_REACHED_GCOVR_EXCL_LINE; } - buildMatcher(proto_config.match_config(), matchers_); + envoy::config::common::matcher::v3::MatchPredicate match; + if (proto_config.has_match()) { + // Use the match field whenever it is set. + match = proto_config.match(); + } else if (proto_config.has_match_config()) { + // Fallback to use the deprecated match_config field and upgrade (wire cast) it to the new + // MatchPredicate which is backward compatible with the old MatchPredicate originally + // introduced in the Tap filter. + Config::VersionConverter::upgrade(proto_config.match_config(), match); + } else { + throw EnvoyException(fmt::format("Neither match nor match_config is set in TapConfig: {}", + proto_config.DebugString())); + } + buildMatcher(match, matchers_); } const Matcher& TapConfigBaseImpl::rootMatcher() const { diff --git a/source/extensions/common/tap/tap_config_base.h b/source/extensions/common/tap/tap_config_base.h index 59b53da027f6..8a6014bc143c 100644 --- a/source/extensions/common/tap/tap_config_base.h +++ b/source/extensions/common/tap/tap_config_base.h @@ -7,14 +7,17 @@ #include "envoy/data/tap/v3/common.pb.h" #include "envoy/data/tap/v3/wrapper.pb.h" +#include "extensions/common/matcher/matcher.h" #include "extensions/common/tap/tap.h" -#include "extensions/common/tap/tap_matcher.h" namespace Envoy { namespace Extensions { namespace Common { namespace Tap { +using Matcher = Envoy::Extensions::Common::Matcher::Matcher; +using MatcherPtr = Envoy::Extensions::Common::Matcher::MatcherPtr; + /** * Common utilities for tapping. */ diff --git a/test/extensions/common/matcher/BUILD b/test/extensions/common/matcher/BUILD new file mode 100644 index 000000000000..a2723b48da78 --- /dev/null +++ b/test/extensions/common/matcher/BUILD @@ -0,0 +1,19 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "matcher_test", + srcs = ["matcher_test.cc"], + deps = [ + "//source/extensions/common/matcher:matcher_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/common/matcher/v3:pkg_cc_proto", + ], +) diff --git a/test/extensions/common/tap/tap_matcher_test.cc b/test/extensions/common/matcher/matcher_test.cc similarity index 97% rename from test/extensions/common/tap/tap_matcher_test.cc rename to test/extensions/common/matcher/matcher_test.cc index 9898c7b4ee89..28f6752eb24a 100644 --- a/test/extensions/common/tap/tap_matcher_test.cc +++ b/test/extensions/common/matcher/matcher_test.cc @@ -1,8 +1,8 @@ -#include "envoy/config/tap/v3/common.pb.h" +#include "envoy/config/common/matcher/v3/matcher.pb.h" #include "common/protobuf/utility.h" -#include "extensions/common/tap/tap_matcher.h" +#include "extensions/common/matcher/matcher.h" #include "test/test_common/utility.h" @@ -11,19 +11,19 @@ namespace Envoy { namespace Extensions { namespace Common { -namespace Tap { +namespace Matcher { namespace { -class TapMatcherTestBase { +class MatcherTestBase { public: std::vector matchers_; Matcher::MatchStatusVector statuses_; - envoy::config::tap::v3::MatchPredicate config_; + envoy::config::common::matcher::v3::MatchPredicate config_; enum class Direction { Request, Response }; }; -class TapMatcherTest : public TapMatcherTestBase, public testing::Test { +class TapMatcherTest : public MatcherTestBase, public testing::Test { public: Http::TestRequestHeaderMapImpl request_headers_; Http::TestRequestTrailerMapImpl request_trailers_; @@ -31,12 +31,12 @@ class TapMatcherTest : public TapMatcherTestBase, public testing::Test { Http::TestResponseTrailerMapImpl response_trailers_; }; -class TapMatcherGenericBodyConfigTest : public TapMatcherTestBase, public ::testing::Test {}; +class TapMatcherGenericBodyConfigTest : public MatcherTestBase, public ::testing::Test {}; class TapMatcherGenericBodyTest - : public TapMatcherTestBase, + : public MatcherTestBase, public ::testing::TestWithParam< - std::tuple, std::list>, std::pair>>> { public: @@ -242,8 +242,8 @@ TEST_P(TapMatcherGenericBodyTest, GenericBodyTest) { INSTANTIATE_TEST_SUITE_P( TapMatcherGenericBodyTestSuite, TapMatcherGenericBodyTest, ::testing::Combine( - ::testing::Values(TapMatcherTestBase::Direction::Request, - TapMatcherTestBase::Direction::Response), + ::testing::Values(MatcherTestBase::Direction::Request, + MatcherTestBase::Direction::Response), ::testing::Values( // SEARCHING FOR SINGLE PATTERN - no limit // Should match - there is a single body chunk and envoy is in the body @@ -500,7 +500,7 @@ TEST_F(TapMatcherGenericBodyTest, RandomLengthOverlappingPatterns) { } } } // namespace -} // namespace Tap +} // namespace Matcher } // namespace Common } // namespace Extensions } // namespace Envoy diff --git a/test/extensions/common/tap/BUILD b/test/extensions/common/tap/BUILD index 9775f2873b05..c5a459721faf 100644 --- a/test/extensions/common/tap/BUILD +++ b/test/extensions/common/tap/BUILD @@ -31,16 +31,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "tap_matcher_test", - srcs = ["tap_matcher_test.cc"], - deps = [ - "//source/extensions/common/tap:tap_matcher", - "//test/test_common:utility_lib", - "@envoy_api//envoy/config/tap/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "tap_config_base_test", srcs = ["tap_config_base_test.cc"], diff --git a/test/extensions/common/tap/admin_test.cc b/test/extensions/common/tap/admin_test.cc index 1894251b3bfc..bffe69944cbe 100644 --- a/test/extensions/common/tap/admin_test.cc +++ b/test/extensions/common/tap/admin_test.cc @@ -50,7 +50,7 @@ class AdminHandlerTest : public testing::Test { R"EOF( config_id: test_config_id tap_config: - match_config: + match: any_match: true output_config: sinks: diff --git a/test/extensions/filters/http/tap/BUILD b/test/extensions/filters/http/tap/BUILD index a6b1a6967278..f134caaf5356 100644 --- a/test/extensions/filters/http/tap/BUILD +++ b/test/extensions/filters/http/tap/BUILD @@ -56,6 +56,7 @@ envoy_extension_cc_test( deps = [ "//source/extensions/filters/http/tap:config", "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/data/tap/v3:pkg_cc_proto", ], diff --git a/test/extensions/filters/http/tap/tap_filter_integration_test.cc b/test/extensions/filters/http/tap/tap_filter_integration_test.cc index a68b9d44ad19..fdfa591bc62e 100644 --- a/test/extensions/filters/http/tap/tap_filter_integration_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_integration_test.cc @@ -4,6 +4,7 @@ #include "envoy/data/tap/v3/wrapper.pb.h" #include "test/integration/http_integration.h" +#include "test/test_common/utility.h" #include "absl/strings/match.h" #include "gtest/gtest.h" @@ -127,6 +128,27 @@ class TapIntegrationTest : public testing::TestWithParamclose(); + test_server_->waitForCounterGe("http.config_test.downstream_cx_destroy", 1); + + // Find the written .pb file and verify it. + auto files = TestUtility::listFiles(path_prefix, false); + auto pb_file = std::find_if(files.begin(), files.end(), + [](const std::string& s) { return absl::EndsWith(s, ".pb"); }); + ASSERT_NE(pb_file, files.end()); + + envoy::data::tap::v3::TraceWrapper trace; + TestUtility::loadFromFile(*pb_file, trace, *api_); + EXPECT_TRUE(trace.has_http_buffered_trace()); + } + const Http::TestRequestHeaderMapImpl request_headers_tap_{{":method", "GET"}, {":path", "/"}, {":scheme", "http"}, @@ -168,10 +190,10 @@ TEST_P(TapIntegrationTest, StaticFilePerTap) { R"EOF( name: tap typed_config: - "@type": type.googleapis.com/envoy.config.filter.http.tap.v2alpha.Tap + "@type": type.googleapis.com/envoy.extensions.filters.http.tap.v3.Tap common_config: static_config: - match_config: + match: any_match: true output_config: sinks: @@ -180,24 +202,53 @@ name: tap path_prefix: {} )EOF"; - const std::string path_prefix = getTempPathPrefix(); - initializeFilter(fmt::format(filter_config, path_prefix)); + verifyStaticFilePerTap(filter_config); +} - // Initial request/response with tap. - codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); - makeRequest(request_headers_tap_, {}, nullptr, response_headers_no_tap_, {}, nullptr); - codec_client_->close(); - test_server_->waitForCounterGe("http.config_test.downstream_cx_destroy", 1); +// Verify the match field takes precedence over the deprecated match_config field. +TEST_P(TapIntegrationTest, DEPRECATED_FEATURE_TEST(StaticFilePerTapWithMatchConfigAndMatch)) { + const std::string filter_config = + R"EOF( +name: tap +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.tap.v3.Tap + common_config: + static_config: + # match_config should be ignored by the match field. + match_config: + not_match: + any_match: true + match: + any_match: true + output_config: + sinks: + - format: PROTO_BINARY + file_per_tap: + path_prefix: {} +)EOF"; - // Find the written .pb file and verify it. - auto files = TestUtility::listFiles(path_prefix, false); - auto pb_file = std::find_if(files.begin(), files.end(), - [](const std::string& s) { return absl::EndsWith(s, ".pb"); }); - ASSERT_NE(pb_file, files.end()); + verifyStaticFilePerTap(filter_config); +} - envoy::data::tap::v3::TraceWrapper trace; - TestUtility::loadFromFile(*pb_file, trace, *api_); - EXPECT_TRUE(trace.has_http_buffered_trace()); +// Verify the deprecated match_config field. +TEST_P(TapIntegrationTest, DEPRECATED_FEATURE_TEST(StaticFilePerTapWithMatchConfig)) { + const std::string filter_config = + R"EOF( +name: tap +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.tap.v3.Tap + common_config: + static_config: + match_config: + any_match: true + output_config: + sinks: + - format: PROTO_BINARY + file_per_tap: + path_prefix: {} +)EOF"; + + verifyStaticFilePerTap(filter_config); } // Verify a basic tap flow using the admin handler. @@ -212,7 +263,7 @@ TEST_P(TapIntegrationTest, AdminBasicFlow) { R"EOF( config_id: test_config_id tap_config: - match_config: + match: or_match: rules: - http_request_headers_match: @@ -275,7 +326,7 @@ config_id: test_config_id R"EOF( config_id: test_config_id tap_config: - match_config: + match: and_match: rules: - http_request_headers_match: @@ -319,7 +370,7 @@ TEST_P(TapIntegrationTest, AdminTrailers) { R"EOF( config_id: test_config_id tap_config: - match_config: + match: and_match: rules: - http_request_trailers_match: @@ -360,7 +411,7 @@ TEST_P(TapIntegrationTest, AdminBodyAsBytes) { R"EOF( config_id: test_config_id tap_config: - match_config: + match: any_match: true output_config: sinks: @@ -391,7 +442,7 @@ TEST_P(TapIntegrationTest, AdminBodyAsString) { R"EOF( config_id: test_config_id tap_config: - match_config: + match: any_match: true output_config: sinks: @@ -423,7 +474,7 @@ TEST_P(TapIntegrationTest, AdminBodyAsBytesTruncated) { R"EOF( config_id: test_config_id tap_config: - match_config: + match: any_match: true output_config: max_buffered_rx_bytes: 3 @@ -546,7 +597,7 @@ TEST_P(TapIntegrationTest, AdminBodyMatching) { R"EOF( config_id: test_config_id tap_config: - match_config: + match: and_match: rules: - http_request_generic_body_match: diff --git a/test/extensions/filters/http/tap/tap_filter_test.cc b/test/extensions/filters/http/tap/tap_filter_test.cc index c4d229fd192d..7ed0f1afb77f 100644 --- a/test/extensions/filters/http/tap/tap_filter_test.cc +++ b/test/extensions/filters/http/tap/tap_filter_test.cc @@ -133,7 +133,7 @@ TEST(TapFilterConfigTest, InvalidProto) { R"EOF( common_config: static_config: - match_config: + match: any_match: true output_config: sinks: @@ -150,6 +150,29 @@ TEST(TapFilterConfigTest, InvalidProto) { "Error: Specifying admin streaming output without configuring admin."); } +TEST(TapFilterConfigTest, NeitherMatchNorMatchConfig) { + const std::string filter_config = + R"EOF( + common_config: + static_config: + output_config: + sinks: + - format: PROTO_BINARY + file_per_tap: + path_prefix: abc +)EOF"; + + envoy::extensions::filters::http::tap::v3::Tap config; + TestUtility::loadFromYaml(filter_config, config); + NiceMock context; + TapFilterFactory factory; + + EXPECT_THROW_WITH_MESSAGE(factory.createFilterFactoryFromProto(config, "stats", context), + EnvoyException, + fmt::format("Neither match nor match_config is set in TapConfig: {}", + config.common_config().static_config().DebugString())); +} + } // namespace } // namespace TapFilter } // namespace HttpFilters diff --git a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc index 9994f8ca314b..db9b0afd9ec5 100644 --- a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc +++ b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc @@ -396,10 +396,8 @@ class SslTapIntegrationTest : public SslIntegrationTest { envoy::extensions::transport_sockets::tap::v3::Tap createTapConfig(const envoy::config::core::v3::TransportSocket& inner_transport) { envoy::extensions::transport_sockets::tap::v3::Tap tap_config; - tap_config.mutable_common_config() - ->mutable_static_config() - ->mutable_match_config() - ->set_any_match(true); + tap_config.mutable_common_config()->mutable_static_config()->mutable_match()->set_any_match( + true); auto* output_config = tap_config.mutable_common_config()->mutable_static_config()->mutable_output_config(); if (max_rx_bytes_.has_value()) { From a520aeae8e2239ab0aea26a967317ef38cc86be8 Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 10 Aug 2020 13:35:54 -0400 Subject: [PATCH 16/67] quiche: verify private key used to sign in ComputeTlsSignature() (#12553) quic::ProofSource:: ComputeTlsSignature() can be called by either ProofSource::GetProof() or directly from QUICHE stack. When it's called from GetProof() the private key is verified while deducing signature algorithm in GetProof(). But when it is called from QUICHE TLS handshake stack directly, we currently don't check the private key used. Fixes #9434 Part of #2557 Signed-off-by: Dan Zhang --- .../quiche/envoy_quic_proof_source.cc | 12 ++++++- .../quiche/envoy_quic_proof_source_test.cc | 36 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc index 96fe056e818e..1f65e4e7e6a0 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_proof_source.cc @@ -58,9 +58,19 @@ void EnvoyQuicProofSource::signPayload( callback->Run(false, "", nullptr); return; } + // Verify the signature algorithm is as expected. + std::string error_details; + int sign_alg = deduceSignatureAlgorithmFromPublicKey(pem_key->private_key(), &error_details); + if (sign_alg != signature_algorithm) { + ENVOY_LOG(warn, + fmt::format("The signature algorithm {} from the private key is not expected: {}", + sign_alg, error_details)); + callback->Run(false, "", nullptr); + return; + } + // Sign. std::string sig = pem_key->Sign(in, signature_algorithm); - bool success = !sig.empty(); ASSERT(res.filter_chain_.has_value()); callback->Run(success, sig, diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc index d896dbb86b7c..cbf66f511f50 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc @@ -242,6 +242,42 @@ GUy+n0vQNB0cXGzgcGI= testGetProof(false); } +TEST_F(EnvoyQuicProofSourceTest, UnexpectedPrivateKey) { + EXPECT_CALL(listen_socket_, ioHandle()); + EXPECT_CALL(filter_chain_manager_, findFilterChain(_)) + .WillOnce(Invoke([&](const Network::ConnectionSocket&) { return &filter_chain_; })); + auto server_context_config = std::make_unique(); + auto server_context_config_ptr = server_context_config.get(); + QuicServerTransportSocketFactory transport_socket_factory(std::move(server_context_config)); + EXPECT_CALL(filter_chain_, transportSocketFactory()) + .WillRepeatedly(ReturnRef(transport_socket_factory)); + + Ssl::MockTlsCertificateConfig tls_cert_config; + std::vector> tls_cert_configs{ + std::reference_wrapper(tls_cert_config)}; + EXPECT_CALL(*server_context_config_ptr, tlsCertificates()) + .WillRepeatedly(Return(tls_cert_configs)); + std::string rsa_pkey_1024_len(R"(-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQC79hDq/OwN3ke3EF6Ntdi9R+VSrl9MStk992l1us8lZhq+e0zU +OlvxbUeZ8wyVkzs1gqI1it1IwF+EpdGhHhjggZjg040GD3HWSuyCzpHh+nLwJxtQ +D837PCg0zl+TnKv1YjY3I1F3trGhIqfd2B6pgaJ4hpr+0hdqnKP0Htd4DwIDAQAB +AoGASNypUD59Tx70k+1fifWNMEq3heacgJmfPxsyoXWqKSg8g8yOStLYo20mTXJf +VXg+go7CTJkpELOqE2SoL5nYMD0D/YIZCgDx85k0GWHdA6udNn4to95ZTeZPrBHx +T0QNQHnZI3A7RwLinO60IRY0NYzhkTEBxIuvIY6u0DVbrAECQQDpshbxK3DHc7Yi +Au7BUsxP8RbG4pP5IIVoD4YvJuwUkdrfrwejqTdkfchJJc+Gu/+h8vy7eASPHLLT +NBk5wFoPAkEAzeaKnx0CgNs0RX4+sSF727FroD98VUM38OFEJQ6U9OAWGvaKd8ey +yAYUjR2Sl5ZRyrwWv4IqyWgUGhZqNG0CAQJAPTjjm8DGpenhcB2WkNzxG4xMbEQV +gfGMIYvXmmi29liTn4AKH00IbvIo00jtih2cRcATh8VUZG2fR4dhiGik7wJAWSwS +NwzaS7IjtkERp6cHvELfiLxV/Zsp/BGjcKUbD96I1E6X834ySHyRo/f9x9bbP4Es +HO6j1yxTIGU6w8++AQJACdFPnRidOaj5oJmcZq0s6WGTYfegjTOKgi5KQzO0FTwG +qGm130brdD+1U1EJnEFmleLZ/W6mEi3MxcKpWOpTqQ== +-----END RSA PRIVATE KEY-----)"); + EXPECT_CALL(tls_cert_config, privateKey()).WillOnce(ReturnRef(rsa_pkey_1024_len)); + proof_source_.ComputeTlsSignature(server_address_, client_address_, hostname_, + SSL_SIGN_RSA_PSS_RSAE_SHA256, "payload", + std::make_unique(false)); +} + TEST_F(EnvoyQuicProofSourceTest, InvalidPrivateKey) { EXPECT_CALL(listen_socket_, ioHandle()); EXPECT_CALL(filter_chain_manager_, findFilterChain(_)) From 67c5b435fd4723198091a658d913823b586b3854 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 10 Aug 2020 13:39:42 -0400 Subject: [PATCH 17/67] [fuzz] fix bad inputs and config bugs (#12504) * Unset metadata matcher causes crash, since no matcher is set * `encodeHeaders` in HCM requires that the only 1xx header is a 101 upgrade * stream info destructor issue -- this one no longer reproduces Fixes: https://oss-fuzz.com/testcase?key=4863844862918656 https://oss-fuzz.com/testcase-detail/5656400764862464 https://oss-fuzz.com/testcase-detail/5631179290836992 Signed-off-by: Asra Ali --- source/common/access_log/access_log_impl.cc | 18 +++++++------ .../common/access_log/access_log_impl_test.cc | 26 +++++++++++++++++++ .../http/conn_manager_impl_corpus/status_163 | 22 ++++++++++++++++ .../http/conn_manager_impl_fuzz_test.cc | 8 +++++- .../fuzz/filter_corpus/metadata_not_reached | 7 +++++ .../h1_corpus/stream_info_destructor | 24 +++++++++++++++++ 6 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 test/common/http/conn_manager_impl_corpus/status_163 create mode 100644 test/extensions/filters/http/common/fuzz/filter_corpus/metadata_not_reached create mode 100644 test/integration/h1_corpus/stream_info_destructor diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index db4b8330370c..447f951bc2f8 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -262,15 +262,17 @@ MetadataFilter::MetadataFilter(const envoy::config::accesslog::v3::MetadataFilte : default_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(filter_config, match_if_key_not_found, true)), filter_(filter_config.matcher().filter()) { - auto& matcher_config = filter_config.matcher(); + if (filter_config.has_matcher()) { + auto& matcher_config = filter_config.matcher(); - for (const auto& seg : matcher_config.path()) { - path_.push_back(seg.key()); - } + for (const auto& seg : matcher_config.path()) { + path_.push_back(seg.key()); + } - // Matches if the value equals the configured 'MetadataMatcher' value. - const auto& val = matcher_config.value(); - value_matcher_ = Matchers::ValueMatcher::create(val); + // Matches if the value equals the configured 'MetadataMatcher' value. + const auto& val = matcher_config.value(); + value_matcher_ = Matchers::ValueMatcher::create(val); + } // Matches if the value is present in dynamic metadata auto present_val = envoy::type::matcher::v3::ValueMatcher(); @@ -286,7 +288,7 @@ bool MetadataFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::Re // If the key corresponds to a set value in dynamic metadata, return true if the value matches the // the configured 'MetadataMatcher' value and false otherwise if (present_matcher_->match(value)) { - return value_matcher_->match(value); + return value_matcher_ && value_matcher_->match(value); } // If the key does not correspond to a set value in dynamic metadata, return true if diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 2c882010c657..788d8885ba50 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -1310,6 +1310,32 @@ name: accesslog EXPECT_CALL(*file_, write(_)).Times(0); } +// This is a regression test for fuzz bug https://oss-fuzz.com/testcase-detail/4863844862918656 +// where a missing matcher would attempt to create a ValueMatcher and crash in debug mode. Instead, +// the configured metadata filter does not match. +TEST_F(AccessLogImplTest, MetadataFilterNoMatcher) { + const std::string yaml = R"EOF( +name: accesslog +filter: + metadata_filter: + match_if_key_not_found: false +typed_config: + "@type": type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog + path: /dev/null + )EOF"; + + TestStreamInfo stream_info; + ProtobufWkt::Struct metadata_val; + stream_info.setDynamicMetadata("some.namespace", metadata_val); + + const InstanceSharedPtr log = + AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_); + + // If no matcher is set, then expect no logs. + EXPECT_CALL(*file_, write(_)).Times(0); + log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info); +} + TEST_F(AccessLogImplTest, MetadataFilterNoKey) { const std::string default_true_yaml = R"EOF( name: accesslog diff --git a/test/common/http/conn_manager_impl_corpus/status_163 b/test/common/http/conn_manager_impl_corpus/status_163 new file mode 100644 index 000000000000..3c8e7c99f56f --- /dev/null +++ b/test/common/http/conn_manager_impl_corpus/status_163 @@ -0,0 +1,22 @@ +actions { + new_stream { + request_headers { + headers { + key: ":path" + value: "/" + } + } + } +} +actions { + stream_action { + response { + headers { + headers { + key: ":status" + value: "162" + } + } + } + } +} \ No newline at end of file diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 4a7315cf56c4..99f8b18dc779 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -477,11 +477,17 @@ class FuzzStream { Fuzz::fromHeaders(response_action.headers())); // The client codec will ensure we always have a valid :status. // Similarly, local replies should always contain this. + uint64_t status; try { - Utility::getResponseStatus(*headers); + status = Utility::getResponseStatus(*headers); } catch (const CodecClientException&) { headers->setReferenceKey(Headers::get().Status, "200"); } + // The only 1xx header that may be provided to encodeHeaders() is a 101 upgrade, + // guaranteed by the codec parsers. See include/envoy/http/filter.h. + if (CodeUtility::is1xx(status) && status != enumToInt(Http::Code::SwitchingProtocols)) { + headers->setReferenceKey(Headers::get().Status, "200"); + } decoder_filter_->callbacks_->encodeHeaders(std::move(headers), end_stream); state = end_stream ? StreamState::Closed : StreamState::PendingDataOrTrailers; } diff --git a/test/extensions/filters/http/common/fuzz/filter_corpus/metadata_not_reached b/test/extensions/filters/http/common/fuzz/filter_corpus/metadata_not_reached new file mode 100644 index 000000000000..0e714ec32f42 --- /dev/null +++ b/test/extensions/filters/http/common/fuzz/filter_corpus/metadata_not_reached @@ -0,0 +1,7 @@ +config { + name: "envoy.router" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + value: "\020\001\032\200\005\n\0012\022\372\004:\367\004\022\207\004:\204\004\022\255\001:\252\001\022\004\n\002\n\000\022[2Y\n\006\n\004\n\002\010\001\nK:I\022927\n\002R\000\n12/\n):\'\022\tB\007\n\005\n\001)@\001\022\0222\020\n\006\n\004\n\002\010\002\n\002\032\000\n\002b\000\022\006\n\004\n\002\010\001\n\002\032\000\022\006\n\004\n\002\010\001\022\004\n\002\n\000\n\002\032\000\022\006\n\004\n\002\010\001\022725\n\004\n\002\n\000\n\002\032\000\n)2\'\n!:\037\022\tB\007\n\005\n\001)@\001\022\n2\010\n\002\032\000\n\002\"\000\022\006\n\004\n\002\010\001\n\002\032\000\022\004\n\002\n\000\022\004\n\002\n\000\022\263\0022\260\002\n\002R\000\n\245\0022\242\002\n\206\0022\203\002\nw2u\nj:h\022\004\n\002\n\000\022@2>\n8:6\022&2$\n\002R\000\n\0362\034\n\026:\024\022\n2\010\n\002\032\000\n\002\"\000\022\006\n\004\n\002\010\001\n\002\032\000\022\006\n\004\n\002\010\001\022\004\n\002\n\000\n\002\032\000\022\006\n\004\n\002\010\001\022\0202\016\n\004\n\002\n\000\n\002\032\000\n\002\"\000\022\004\n\002\n\000\n\007R\005\n\001\002\020\001\n.:,\022\0342\032\n\005R\003\n\001\004\n\r2\013\n\005R\003\n\001\002\n\002\032\000\n\002\032\000\022\006\n\004\n\002\010\001\022\004\n\002\n\000\nR:P\022<2:\n\006\n\004\n\002\010\001\n,:*\022\0342\032\n\005R\003\n\001\004\n\r2\013\n\005R\003\n\001\002\n\002\032\000\n\002\032\000\022\004\n\002\n\000\022\004\n\002\n\000\n\002\032\000\022\006\n\004\n\002\010\001\022\002\"\000\022\004\n\002\n\000\n\004R\002\020\001\n\027:\025\022\r2\013\n\005R\003\n\001\004\n\002\032\000\022\004\n\002\n\000\n\002\032\000\022\006\n\004\n\002\010\001\022\006\n\004\n\002\010\001\022\006\n\004\n\002\010\002\022\002\032\000\022J*H\nD\n\000\032@\022>2<\n:28\n6:4\022220\n.2,\n*2(\n&:$\022\"2 \n\000\n\034:\032\022\0302\026\n\n2\010\n\006:\004\022\0022\000\n\010:\006\022\0042\002\n\000\030\001\022\033:\031\022\002J\000\022\013\n\t\n\007\010\001\022\003\032\001/\022\006\n\004\n\002\010\002\032(\n\016\177\177\177\177\177\177\177\177\177\177\177\177\177\177\022\0262\024\n\002R\000\n\n2\010\n\002R\000\n\002\032\000\n\002\032\0000\001" + } +} \ No newline at end of file diff --git a/test/integration/h1_corpus/stream_info_destructor b/test/integration/h1_corpus/stream_info_destructor new file mode 100644 index 000000000000..63f9a21a8fb7 --- /dev/null +++ b/test/integration/h1_corpus/stream_info_destructor @@ -0,0 +1,24 @@ +events { + downstream_send_bytes: "POST /test/long/url HTTP/1.1\r\nhost: host\r\nx-lyft-user-id: -063%\nuser-agent: /4302450943\n\t\t08856android\363\243x-lyft-user-id: -063%\nuser-agent: /4;02450943\n\t\t08856android\363\243\201$80\n\t\t\t\n\t\t\t\tAe1\201\24180\n\t\t\t\n\t\t\t\tAe118\tefts " +} +events { +} +events { + downstream_send_bytes: "POST //urk HTTP/1.1\r\nshhfot: ost\r\n -253%\nuser-agent: /0%\nuser-agent: /430%\nuser-agent:4967:18446744073709551615iOS~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~>~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\201~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~a~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-7749978774642053139~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttuttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt\326Utttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt-4017153681670550988tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt|tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttstttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt\364ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt\364ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttaaaaaaaaaaaaaaaaaaaaaaa-6742158280474489582aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\r\n" +} +events { +} +events { +} +events { + downstream_send_bytes: "POST /test/lon\nte: e: h" +} +events { +} +events { + downstream_send_bytes: "POST //urk HTTP/1.1\r\nshhfot: ost\r\n -253%\nuser-agent: /0%\nuser-agent: /430%\nuser-agent:4967:18446744073709551615iOS~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~>~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\201~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~a~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~-7749978774642053139~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttuttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt + +tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt\326Utttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt-4017153681670550988tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt|tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttstttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt\364ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt\364ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttaaaaaaaaaaaaaaaaaaaaaaa-6742158280474489582aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n\r\n" +} +events { +} \ No newline at end of file From cf850688a93b8dea5705170a211ea98b77beda61 Mon Sep 17 00:00:00 2001 From: Arthur Yan <55563955+arthuryan-k@users.noreply.github.com> Date: Mon, 10 Aug 2020 14:01:28 -0400 Subject: [PATCH 18/67] fuzz: added fuzz test for listener filter http_inspector (#12411) Commit Message: Added fuzz test for listener filter Additional Description: Extended generic listener filter fuzzer library to support mocked dispatcher and system call behavior Created http_inspector_corpus and populated with testcases (valid and invalid headers) Created http_inspector_fuzz_test.cc and updated ListenerFilterFuzzer API Signed-off-by: Arthur Yan --- .../filters/listener/common/fuzz/BUILD | 13 ++- .../common/fuzz/listener_filter_fakes.cc | 86 ++++++++++++++++++ .../common/fuzz/listener_filter_fakes.h | 70 +++++++++++++++ .../common/fuzz/listener_filter_fuzzer.cc | 87 +++++++++++++++++-- .../common/fuzz/listener_filter_fuzzer.h | 39 +++++++-- .../common/fuzz/listener_filter_fuzzer.proto | 1 + .../filters/listener/http_inspector/BUILD | 11 +++ .../http_inspector_corpus/bad_header | 1 + .../http_inspector_corpus/incomplete_header | 1 + .../http_inspector_corpus/invalid_method | 1 + .../http_inspector_corpus/invalid_request | 1 + .../http_inspector_corpus/multiple_http10 | 3 + .../http_inspector_corpus/multiple_incomplete | 2 + .../http_inspector_corpus/valid_http10 | 1 + .../http_inspector_corpus/valid_http11 | 1 + .../http_inspector_corpus/valid_http2 | 1 + .../http_inspector_fuzz_test.cc | 32 +++++++ .../original_dst_corpus/invalid_scheme | 3 + .../original_dst_corpus/invalid_test | 4 - .../{unix_test => invalid_unix} | 1 - .../{ipv4_test => valid_ipv4} | 1 - .../{ipv6_test => valid_ipv6} | 1 - .../original_dst/original_dst_fuzz_test.cc | 9 +- .../{ipv4_test => valid_ipv4} | 7 +- .../{unix_test => valid_unix} | 5 +- .../original_src/original_src_fuzz_test.cc | 10 +-- .../original_src/original_src_fuzz_test.proto | 2 +- test/mocks/network/BUILD | 8 -- test/mocks/network/fakes.h | 62 ------------- 29 files changed, 346 insertions(+), 118 deletions(-) create mode 100644 test/extensions/filters/listener/common/fuzz/listener_filter_fakes.cc create mode 100644 test/extensions/filters/listener/common/fuzz/listener_filter_fakes.h create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/bad_header create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/incomplete_header create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_method create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_request create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_http10 create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_incomplete create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http10 create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http11 create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http2 create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_fuzz_test.cc create mode 100644 test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_scheme delete mode 100644 test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_test rename test/extensions/filters/listener/original_dst/original_dst_corpus/{unix_test => invalid_unix} (55%) rename test/extensions/filters/listener/original_dst/original_dst_corpus/{ipv4_test => valid_ipv4} (54%) rename test/extensions/filters/listener/original_dst/original_dst_corpus/{ipv6_test => valid_ipv6} (56%) rename test/extensions/filters/listener/original_src/original_src_corpus/{ipv4_test => valid_ipv4} (53%) rename test/extensions/filters/listener/original_src/original_src_corpus/{unix_test => valid_unix} (61%) delete mode 100644 test/mocks/network/fakes.h diff --git a/test/extensions/filters/listener/common/fuzz/BUILD b/test/extensions/filters/listener/common/fuzz/BUILD index e306f3a43382..85ed3cbf7304 100644 --- a/test/extensions/filters/listener/common/fuzz/BUILD +++ b/test/extensions/filters/listener/common/fuzz/BUILD @@ -19,9 +19,20 @@ envoy_cc_test_library( srcs = ["listener_filter_fuzzer.cc"], hdrs = ["listener_filter_fuzzer.h"], deps = [ + ":listener_filter_fakes", ":listener_filter_fuzzer_proto_cc_proto", "//include/envoy/network:filter_interface", - "//test/mocks/network:network_fakes", + "//test/mocks/network:network_mocks", + "//test/test_common:threadsafe_singleton_injector_lib", + ], +) + +envoy_cc_test_library( + name = "listener_filter_fakes", + srcs = ["listener_filter_fakes.cc"], + hdrs = ["listener_filter_fakes.h"], + deps = [ + "//source/common/api:os_sys_calls_lib", "//test/mocks/network:network_mocks", ], ) diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fakes.cc b/test/extensions/filters/listener/common/fuzz/listener_filter_fakes.cc new file mode 100644 index 000000000000..f0546c7950fe --- /dev/null +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fakes.cc @@ -0,0 +1,86 @@ +#include "test/extensions/filters/listener/common/fuzz/listener_filter_fakes.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { + +Network::IoHandle& FakeConnectionSocket::ioHandle() { return *io_handle_; } + +const Network::IoHandle& FakeConnectionSocket::ioHandle() const { return *io_handle_; } + +void FakeConnectionSocket::setLocalAddress( + const Network::Address::InstanceConstSharedPtr& local_address) { + local_address_ = local_address; + if (local_address_ != nullptr) { + addr_type_ = local_address_->type(); + } +} + +void FakeConnectionSocket::setRemoteAddress( + const Network::Address::InstanceConstSharedPtr& remote_address) { + remote_address_ = remote_address; +} + +const Network::Address::InstanceConstSharedPtr& FakeConnectionSocket::localAddress() const { + return local_address_; +} + +const Network::Address::InstanceConstSharedPtr& FakeConnectionSocket::remoteAddress() const { + return remote_address_; +} + +Network::Address::Type FakeConnectionSocket::addressType() const { return addr_type_; } + +absl::optional FakeConnectionSocket::ipVersion() const { + if (local_address_ == nullptr || addr_type_ != Network::Address::Type::Ip) { + return absl::nullopt; + } + + return local_address_->ip()->version(); +} + +void FakeConnectionSocket::setDetectedTransportProtocol(absl::string_view protocol) { + transport_protocol_ = std::string(protocol); +} + +absl::string_view FakeConnectionSocket::detectedTransportProtocol() const { + return transport_protocol_; +} + +void FakeConnectionSocket::setRequestedApplicationProtocols( + const std::vector& protocols) { + application_protocols_.clear(); + for (const auto& protocol : protocols) { + application_protocols_.emplace_back(protocol); + } +} + +const std::vector& FakeConnectionSocket::requestedApplicationProtocols() const { + return application_protocols_; +} + +void FakeConnectionSocket::setRequestedServerName(absl::string_view server_name) { + server_name_ = std::string(server_name); +} + +absl::string_view FakeConnectionSocket::requestedServerName() const { return server_name_; } + +Api::SysCallIntResult FakeConnectionSocket::getSocketOption(int level, int, void* optval, + socklen_t*) const { + switch (level) { + case SOL_IPV6: + static_cast(optval)->ss_family = AF_INET6; + break; + case SOL_IP: + static_cast(optval)->ss_family = AF_INET; + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + + return Api::SysCallIntResult{0, 0}; +} + +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fakes.h b/test/extensions/filters/listener/common/fuzz/listener_filter_fakes.h new file mode 100644 index 000000000000..4e13b4e6f418 --- /dev/null +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fakes.h @@ -0,0 +1,70 @@ +#include "common/api/os_sys_calls_impl.h" +#include "common/network/io_socket_handle_impl.h" + +#include "test/mocks/network/mocks.h" + +#include "gmock/gmock.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { + +static constexpr int kFakeSocketFd = 42; + +class FakeConnectionSocket : public Network::MockConnectionSocket { +public: + FakeConnectionSocket() + : io_handle_(std::make_unique(kFakeSocketFd)), + local_address_(nullptr), remote_address_(nullptr) {} + + ~FakeConnectionSocket() override { io_handle_->close(); } + + Network::IoHandle& ioHandle() override; + + const Network::IoHandle& ioHandle() const override; + + void setLocalAddress(const Network::Address::InstanceConstSharedPtr& local_address) override; + + void setRemoteAddress(const Network::Address::InstanceConstSharedPtr& remote_address) override; + + const Network::Address::InstanceConstSharedPtr& localAddress() const override; + + const Network::Address::InstanceConstSharedPtr& remoteAddress() const override; + + Network::Address::Type addressType() const override; + + absl::optional ipVersion() const override; + + void setRequestedApplicationProtocols(const std::vector& protocols) override; + + const std::vector& requestedApplicationProtocols() const override; + + void setDetectedTransportProtocol(absl::string_view protocol) override; + + absl::string_view detectedTransportProtocol() const override; + + void setRequestedServerName(absl::string_view server_name) override; + + absl::string_view requestedServerName() const override; + + Api::SysCallIntResult getSocketOption(int level, int, void* optval, socklen_t*) const override; + +private: + const Network::IoHandlePtr io_handle_; + Network::Address::InstanceConstSharedPtr local_address_; + Network::Address::InstanceConstSharedPtr remote_address_; + Network::Address::Type addr_type_; + std::vector application_protocols_; + std::string transport_protocol_; + std::string server_name_; +}; + +// TODO: Move over to Fake (name is confusing) +class FakeOsSysCalls : public Api::OsSysCallsImpl { +public: + MOCK_METHOD(Api::SysCallSizeResult, recv, (os_fd_t, void*, size_t, int)); +}; + +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc index f38aba8918f2..0f5aa60b8d44 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc @@ -1,7 +1,5 @@ #include "test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h" -#include "common/network/utility.h" - namespace Envoy { namespace Extensions { namespace ListenerFilters { @@ -10,21 +8,92 @@ void ListenerFilterFuzzer::fuzz( Network::ListenerFilter& filter, const test::extensions::filters::listener::FilterFuzzTestCase& input) { try { - fuzzerSetup(input); + socket_.setLocalAddress(Network::Utility::resolveUrl(input.sock().local_address())); } catch (const EnvoyException& e) { - ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); - return; + // Socket's local address will be nullptr by default if fuzzed local address is malformed + // or missing - local address field in proto is optional + } + try { + socket_.setRemoteAddress(Network::Utility::resolveUrl(input.sock().remote_address())); + } catch (const EnvoyException& e) { + // Socket's remote address will be nullptr by default if fuzzed remote address is malformed + // or missing - remote address field in proto is optional + } + + FuzzedHeader header(input); + + if (!header.empty()) { + ON_CALL(os_sys_calls_, recv(kFakeSocketFd, _, _, MSG_PEEK)) + .WillByDefault(testing::Return(Api::SysCallSizeResult{static_cast(0), 0})); + + ON_CALL(dispatcher_, + createFileEvent_(_, _, Event::FileTriggerType::Edge, + Event::FileReadyType::Read | Event::FileReadyType::Closed)) + .WillByDefault(testing::DoAll(testing::SaveArg<1>(&file_event_callback_), + testing::ReturnNew>())); } filter.onAccept(cb_); + + if (file_event_callback_ == nullptr) { + // If filter does not call createFileEvent (i.e. original_dst and original_src) + return; + } + + if (!header.empty()) { + { + testing::InSequence s; + + EXPECT_CALL(os_sys_calls_, recv(kFakeSocketFd, _, _, MSG_PEEK)) + .Times(testing::AnyNumber()) + .WillRepeatedly(Invoke( + [&header](os_fd_t, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + return header.next(buffer, length); + })); + } + + bool got_continue = false; + + ON_CALL(cb_, continueFilterChain(true)) + .WillByDefault(testing::InvokeWithoutArgs([&got_continue]() { got_continue = true; })); + + while (!got_continue) { + if (header.done()) { // End of stream reached but not done + file_event_callback_(Event::FileReadyType::Closed); + } else { + file_event_callback_(Event::FileReadyType::Read); + } + } + } } -void ListenerFilterFuzzer::socketSetup( - const test::extensions::filters::listener::FilterFuzzTestCase& input) { - socket_.setLocalAddress(Network::Utility::resolveUrl(input.sock().local_address())); - socket_.setRemoteAddress(Network::Utility::resolveUrl(input.sock().remote_address())); +FuzzedHeader::FuzzedHeader(const test::extensions::filters::listener::FilterFuzzTestCase& input) + : nreads_(input.data_size()), nread_(0) { + size_t len = 0; + for (int i = 0; i < nreads_; i++) { + len += input.data(i).size(); + } + + header_.reserve(len); + + for (int i = 0; i < nreads_; i++) { + header_ += input.data(i); + indices_.push_back(header_.size()); + } } +Api::SysCallSizeResult FuzzedHeader::next(void* buffer, size_t length) { + if (done()) { // End of stream reached + nread_ = nreads_ - 1; // Decrement to avoid out-of-range for last recv() call + } + memcpy(buffer, header_.data(), std::min(indices_[nread_], length)); + return Api::SysCallSizeResult{static_cast(indices_[nread_++]), 0}; +} + +bool FuzzedHeader::done() { return nread_ >= nreads_; } + +bool FuzzedHeader::empty() { return nreads_ == 0; } + } // namespace ListenerFilters } // namespace Extensions } // namespace Envoy diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h index fe81a9e12cc4..66b6f8707bfd 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h @@ -1,8 +1,10 @@ #include "envoy/network/filter.h" +#include "test/extensions/filters/listener/common/fuzz/listener_filter_fakes.h" #include "test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.pb.validate.h" -#include "test/mocks/network/fakes.h" +#include "test/mocks/event/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "gmock/gmock.h" @@ -12,19 +14,40 @@ namespace ListenerFilters { class ListenerFilterFuzzer { public: + ListenerFilterFuzzer() { + ON_CALL(cb_, socket()).WillByDefault(testing::ReturnRef(socket_)); + ON_CALL(cb_, dispatcher()).WillByDefault(testing::ReturnRef(dispatcher_)); + } + void fuzz(Network::ListenerFilter& filter, const test::extensions::filters::listener::FilterFuzzTestCase& input); private: - void fuzzerSetup(const test::extensions::filters::listener::FilterFuzzTestCase& input) { - ON_CALL(cb_, socket()).WillByDefault(testing::ReturnRef(socket_)); - socketSetup(input); - } + FakeOsSysCalls os_sys_calls_; + TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; + NiceMock cb_; + FakeConnectionSocket socket_; + NiceMock dispatcher_; + Event::FileReadyCb file_event_callback_; +}; - void socketSetup(const test::extensions::filters::listener::FilterFuzzTestCase& input); +class FuzzedHeader { +public: + FuzzedHeader(const test::extensions::filters::listener::FilterFuzzTestCase& input); - NiceMock cb_; - Network::FakeConnectionSocket socket_; + // Copies next read into buffer and returns the number of bytes written + Api::SysCallSizeResult next(void* buffer, size_t length); + + bool done(); + + // Returns true if data field in proto is empty + bool empty(); + +private: + const int nreads_; // Number of reads + int nread_; // Counter of current read + std::string header_; // Construct header from single or multiple reads + std::vector indices_; // Ending indices for each read }; } // namespace ListenerFilters diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto index 916c645d41ba..5741ed9edfa3 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto @@ -9,4 +9,5 @@ message Socket { message FilterFuzzTestCase { Socket sock = 1; + repeated string data = 2; } \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/BUILD b/test/extensions/filters/listener/http_inspector/BUILD index 8530e24434d9..05f898a7bf90 100644 --- a/test/extensions/filters/listener/http_inspector/BUILD +++ b/test/extensions/filters/listener/http_inspector/BUILD @@ -1,5 +1,6 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_fuzz_test", "envoy_package", ) load( @@ -42,3 +43,13 @@ envoy_extension_cc_test( "//test/test_common:threadsafe_singleton_injector_lib", ], ) + +envoy_cc_fuzz_test( + name = "http_inspector_fuzz_test", + srcs = ["http_inspector_fuzz_test.cc"], + corpus = "http_inspector_corpus", + deps = [ + "//source/extensions/filters/listener/http_inspector:http_inspector_lib", + "//test/extensions/filters/listener/common/fuzz:listener_filter_fuzzer_lib", + ], +) diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/bad_header b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/bad_header new file mode 100644 index 000000000000..a84991228ff5 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/bad_header @@ -0,0 +1 @@ +data: "X" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/incomplete_header b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/incomplete_header new file mode 100644 index 000000000000..db337b0c762a --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/incomplete_header @@ -0,0 +1 @@ +data: "GE" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_method b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_method new file mode 100644 index 000000000000..b14ffd72e116 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_method @@ -0,0 +1 @@ +data: "BAD /anything HTTP/1.1" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_request b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_request new file mode 100644 index 000000000000..a7943ddb30b1 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/invalid_request @@ -0,0 +1 @@ +data: "BAD /anything HTTP/1.1\r\n" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_http10 b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_http10 new file mode 100644 index 000000000000..42fa7434ebbb --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_http10 @@ -0,0 +1,3 @@ +data: "GET /anyt" +data: "hing HT" +data: "TP/1.0\r" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_incomplete b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_incomplete new file mode 100644 index 000000000000..58c5d8ad8613 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/multiple_incomplete @@ -0,0 +1,2 @@ +data: "G" +data: "E" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http10 b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http10 new file mode 100644 index 000000000000..5512c5504dd9 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http10 @@ -0,0 +1 @@ +data: "GET /anything HTTP/1.0\r\nhost: google.com\r\nuser-agent: curl/7.64.0\r\naccept: */*\r\nx-forwarded-proto: http\r\nx-request-id: a52df4a0-ed00-4a19-86a7-80e5049c6c84\r\nx-envoy-expected-rq-timeout-ms: 15000\r\ncontent-length: 0\r\n\r\n" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http11 b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http11 new file mode 100644 index 000000000000..56906d74b1c1 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http11 @@ -0,0 +1 @@ +data: "GET /anything HTTP/1.1\r\nhost: google.com\r\nuser-agent: curl/7.64.0\r\naccept: */*\r\nx-forwarded-proto: http\r\nx-request-id: a52df4a0-ed00-4a19-86a7-80e5049c6c84\r\nx-envoy-expected-rq-timeout-ms: 15000\r\ncontent-length: 3\r\n\r\nfoo" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http2 b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http2 new file mode 100644 index 000000000000..0e1faf044c0f --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_corpus/valid_http2 @@ -0,0 +1 @@ +data: "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n\x00\x00\x0c\x04\x00\x00\x00\x00\x00\x00\x04\x10\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x04\x08\x00\x00\x00\x00\x00\x0f\xff\x00\x01\x00\x00}\x01\x05\x00\x00\x00\x01A\x8a\xa0\xe4\x1d\x13\x9d\t\xb8\xf0\x00\x0f\x04\x88`uzL\xe6\xaaf\x05\x82\x86z\x88%\xb6P\xc3\xab\xb8\xd2\xe0S\x03*/*@\x8d\xf2\xb4\xa7\xb3\xc0\xec\x90\xb2-]\x87I\xff\x83\x9d)\xaf@\x89\xf2\xb5\x85\xediP\x95\x8d\'\x9a\x18\x9e\x03\xf1\xcaU\x82&_Y\xa7[\n\xc3\x11\x19Y\xc7\xe4\x90\x04\x90\x8d\xb6\xe8?@\x96\xf2\xb1j\xee\x7fK\x17\xcde\"K\"\xd6vY&\xa4\xa7\xb5+R\x8f\x84\x0b`\x00?" \ No newline at end of file diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_fuzz_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_fuzz_test.cc new file mode 100644 index 000000000000..5f867c22b179 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_fuzz_test.cc @@ -0,0 +1,32 @@ +#include "extensions/filters/listener/http_inspector/http_inspector.h" + +#include "test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h" +#include "test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.pb.validate.h" +#include "test/fuzz/fuzz_runner.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace HttpInspector { + +DEFINE_PROTO_FUZZER(const test::extensions::filters::listener::FilterFuzzTestCase& input) { + + try { + TestUtility::validate(input); + } catch (const ProtoValidationException& e) { + ENVOY_LOG_MISC(debug, "ProtoValidationException: {}", e.what()); + return; + } + + Stats::IsolatedStoreImpl store; + ConfigSharedPtr cfg = std::make_shared(store); + auto filter = std::make_unique(cfg); + + ListenerFilterFuzzer fuzzer; + fuzzer.fuzz(*filter, input); +} + +} // namespace HttpInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_scheme b/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_scheme new file mode 100644 index 000000000000..67994b567f87 --- /dev/null +++ b/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_scheme @@ -0,0 +1,3 @@ +sock { + local_address: "hello world" +} \ No newline at end of file diff --git a/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_test b/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_test deleted file mode 100644 index 7c650514ebbb..000000000000 --- a/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_test +++ /dev/null @@ -1,4 +0,0 @@ -sock { - local_address: "hello world" - remote_address: "tcp://0.0.0.0:0" -} \ No newline at end of file diff --git a/test/extensions/filters/listener/original_dst/original_dst_corpus/unix_test b/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_unix similarity index 55% rename from test/extensions/filters/listener/original_dst/original_dst_corpus/unix_test rename to test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_unix index 3936a8a1c0b1..ee8917b15305 100644 --- a/test/extensions/filters/listener/original_dst/original_dst_corpus/unix_test +++ b/test/extensions/filters/listener/original_dst/original_dst_corpus/invalid_unix @@ -1,4 +1,3 @@ sock { local_address: "unix://tmp/server" - remote_address: "tcp://0.0.0.0:0" } \ No newline at end of file diff --git a/test/extensions/filters/listener/original_dst/original_dst_corpus/ipv4_test b/test/extensions/filters/listener/original_dst/original_dst_corpus/valid_ipv4 similarity index 54% rename from test/extensions/filters/listener/original_dst/original_dst_corpus/ipv4_test rename to test/extensions/filters/listener/original_dst/original_dst_corpus/valid_ipv4 index 20cdd6796db9..a0510b8c253c 100644 --- a/test/extensions/filters/listener/original_dst/original_dst_corpus/ipv4_test +++ b/test/extensions/filters/listener/original_dst/original_dst_corpus/valid_ipv4 @@ -1,4 +1,3 @@ sock { local_address: "tcp://0.0.0.0:0" - remote_address: "tcp://0.0.0.0:0" } \ No newline at end of file diff --git a/test/extensions/filters/listener/original_dst/original_dst_corpus/ipv6_test b/test/extensions/filters/listener/original_dst/original_dst_corpus/valid_ipv6 similarity index 56% rename from test/extensions/filters/listener/original_dst/original_dst_corpus/ipv6_test rename to test/extensions/filters/listener/original_dst/original_dst_corpus/valid_ipv6 index bda8f2989203..32bdadc805ce 100644 --- a/test/extensions/filters/listener/original_dst/original_dst_corpus/ipv6_test +++ b/test/extensions/filters/listener/original_dst/original_dst_corpus/valid_ipv6 @@ -1,4 +1,3 @@ sock { local_address: "tcp://[a:b:c:d::]:0" - remote_address: "tcp://0.0.0.0:0" } \ No newline at end of file diff --git a/test/extensions/filters/listener/original_dst/original_dst_fuzz_test.cc b/test/extensions/filters/listener/original_dst/original_dst_fuzz_test.cc index 5476b6326e3a..4eb1899f3b35 100644 --- a/test/extensions/filters/listener/original_dst/original_dst_fuzz_test.cc +++ b/test/extensions/filters/listener/original_dst/original_dst_fuzz_test.cc @@ -19,13 +19,8 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::listener::FilterFuzzTestCas } auto filter = std::make_unique(); - - try { - ListenerFilterFuzzer fuzzer; - fuzzer.fuzz(*filter, input); - } catch (const EnvoyException& e) { - ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); - } + ListenerFilterFuzzer fuzzer; + fuzzer.fuzz(*filter, input); } } // namespace OriginalDst diff --git a/test/extensions/filters/listener/original_src/original_src_corpus/ipv4_test b/test/extensions/filters/listener/original_src/original_src_corpus/valid_ipv4 similarity index 53% rename from test/extensions/filters/listener/original_src/original_src_corpus/ipv4_test rename to test/extensions/filters/listener/original_src/original_src_corpus/valid_ipv4 index 7d439bab6cad..e9acd000b463 100644 --- a/test/extensions/filters/listener/original_src/original_src_corpus/ipv4_test +++ b/test/extensions/filters/listener/original_src/original_src_corpus/valid_ipv4 @@ -1,11 +1,10 @@ config { - bind_port: true + bind_port: false mark: 0 } -data { +fuzzed { sock { - local_address: "tcp://0.0.0.0:0" remote_address: "tcp://1.2.3.4:0" } -} +} \ No newline at end of file diff --git a/test/extensions/filters/listener/original_src/original_src_corpus/unix_test b/test/extensions/filters/listener/original_src/original_src_corpus/valid_unix similarity index 61% rename from test/extensions/filters/listener/original_src/original_src_corpus/unix_test rename to test/extensions/filters/listener/original_src/original_src_corpus/valid_unix index ecb14359bd52..9726394370c6 100644 --- a/test/extensions/filters/listener/original_src/original_src_corpus/unix_test +++ b/test/extensions/filters/listener/original_src/original_src_corpus/valid_unix @@ -1,11 +1,10 @@ config { bind_port: true - mark: 0 + mark: 15 } -data { +fuzzed { sock { - local_address: "tcp://0.0.0.0:0" remote_address: "unix://domain.socket" } } \ No newline at end of file diff --git a/test/extensions/filters/listener/original_src/original_src_fuzz_test.cc b/test/extensions/filters/listener/original_src/original_src_fuzz_test.cc index c677a55b8d55..0116a7a98b36 100644 --- a/test/extensions/filters/listener/original_src/original_src_fuzz_test.cc +++ b/test/extensions/filters/listener/original_src/original_src_fuzz_test.cc @@ -21,14 +21,8 @@ DEFINE_PROTO_FUZZER( Config config(input.config()); auto filter = std::make_unique(config); - - try { - ListenerFilterFuzzer fuzzer; - fuzzer.fuzz(*filter, input.data()); - } catch (const EnvoyException& e) { - ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); - return; - } + ListenerFilterFuzzer fuzzer; + fuzzer.fuzz(*filter, input.fuzzed()); } } // namespace OriginalSrc diff --git a/test/extensions/filters/listener/original_src/original_src_fuzz_test.proto b/test/extensions/filters/listener/original_src/original_src_fuzz_test.proto index 303b3c86daaa..093378b09045 100644 --- a/test/extensions/filters/listener/original_src/original_src_fuzz_test.proto +++ b/test/extensions/filters/listener/original_src/original_src_fuzz_test.proto @@ -9,6 +9,6 @@ import "validate/validate.proto"; message OriginalSrcTestCase { envoy.extensions.filters.listener.original_src.v3.OriginalSrc config = 1 [(validate.rules).message.required = true]; - test.extensions.filters.listener.FilterFuzzTestCase data = 2 + test.extensions.filters.listener.FilterFuzzTestCase fuzzed = 2 [(validate.rules).message.required = true]; } \ No newline at end of file diff --git a/test/mocks/network/BUILD b/test/mocks/network/BUILD index 5f16adc6206f..020e4b6db404 100644 --- a/test/mocks/network/BUILD +++ b/test/mocks/network/BUILD @@ -63,11 +63,3 @@ envoy_cc_mock( "//source/common/network:utility_lib", ], ) - -envoy_cc_mock( - name = "network_fakes", - hdrs = ["fakes.h"], - deps = [ - ":network_mocks", - ], -) diff --git a/test/mocks/network/fakes.h b/test/mocks/network/fakes.h deleted file mode 100644 index ec69dce0ec0d..000000000000 --- a/test/mocks/network/fakes.h +++ /dev/null @@ -1,62 +0,0 @@ -#include "common/network/utility.h" - -#include "test/mocks/network/mocks.h" - -#include "gmock/gmock.h" - -namespace Envoy { -namespace Network { - -class FakeConnectionSocket : public MockConnectionSocket { -public: - ~FakeConnectionSocket() override = default; - - FakeConnectionSocket() : local_address_(nullptr), remote_address_(nullptr) {} - - FakeConnectionSocket(const Address::InstanceConstSharedPtr& local_address, - const Address::InstanceConstSharedPtr& remote_address) - : local_address_(local_address), remote_address_(remote_address) {} - - void setLocalAddress(const Address::InstanceConstSharedPtr& local_address) override { - local_address_ = local_address; - } - - void setRemoteAddress(const Address::InstanceConstSharedPtr& remote_address) override { - remote_address_ = remote_address; - } - - const Address::InstanceConstSharedPtr& localAddress() const override { return local_address_; } - - const Address::InstanceConstSharedPtr& remoteAddress() const override { return remote_address_; } - - Address::Type addressType() const override { return local_address_->type(); } - - absl::optional ipVersion() const override { - if (local_address_->type() != Address::Type::Ip) { - return absl::nullopt; - } - - return local_address_->ip()->version(); - } - - Api::SysCallIntResult getSocketOption(int level, int, void* optval, socklen_t*) const override { - switch (level) { - case SOL_IPV6: - static_cast(optval)->ss_family = AF_INET6; - break; - case SOL_IP: - static_cast(optval)->ss_family = AF_INET; - break; - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } - - return Api::SysCallIntResult{0, 0}; - } - - Address::InstanceConstSharedPtr local_address_; - Address::InstanceConstSharedPtr remote_address_; -}; - -} // namespace Network -} // namespace Envoy From 366c095e735b47fa4e09f3029f3dc1b2e227a097 Mon Sep 17 00:00:00 2001 From: Sunjay Bhatia Date: Mon, 10 Aug 2020 17:14:43 -0400 Subject: [PATCH 19/67] Windows: Passing unit tests no longer need fails_on_windows tag (#12343) Signed-off-by: William A Rowe Jr Co-authored-by: Sunjay Bhatia --- test/common/http/http2/BUILD | 3 --- test/common/network/BUILD | 2 -- test/common/router/BUILD | 1 - test/common/upstream/BUILD | 2 -- test/extensions/filters/network/rocketmq_proxy/BUILD | 1 - test/server/BUILD | 1 - 6 files changed, 10 deletions(-) diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 467c2785f2aa..6c9e4f8c7f2e 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -45,7 +45,6 @@ envoy_cc_test( "--runtime-feature-override-for-tests=envoy.reloadable_features.new_codec_behavior", ], shard_count = 5, - tags = ["fails_on_windows"], deps = CODEC_TEST_DEPS, ) @@ -57,7 +56,6 @@ envoy_cc_test( "--runtime-feature-disable-for-tests=envoy.reloadable_features.new_codec_behavior", ], shard_count = 5, - tags = ["fails_on_windows"], deps = CODEC_TEST_DEPS, ) @@ -128,7 +126,6 @@ envoy_cc_test( "response_header_corpus/simple_example_huffman", "response_header_corpus/simple_example_plain", ], - tags = ["fails_on_windows"], deps = [ ":frame_replay_lib", "//test/common/http/http2:codec_impl_test_util", diff --git a/test/common/network/BUILD b/test/common/network/BUILD index 0557529c0a0a..3e7b3941d1e3 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -98,7 +98,6 @@ envoy_cc_test( envoy_cc_test( name = "dns_impl_test", srcs = ["dns_impl_test.cc"], - tags = ["fails_on_windows"], deps = [ "//include/envoy/event:dispatcher_interface", "//include/envoy/network:address_interface", @@ -326,7 +325,6 @@ envoy_cc_test( envoy_cc_test( name = "addr_family_aware_socket_option_impl_test", srcs = ["addr_family_aware_socket_option_impl_test.cc"], - tags = ["fails_on_windows"], deps = [ ":socket_option_test", "//source/common/network:addr_family_aware_socket_option_lib", diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 660405054c32..a377e5672fdd 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -289,7 +289,6 @@ envoy_cc_test( name = "router_upstream_log_test", srcs = ["router_upstream_log_test.cc"], external_deps = ["abseil_optional"], - tags = ["fails_on_windows"], deps = [ "//source/common/buffer:buffer_lib", "//source/common/network:utility_lib", diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 9e76ee81c5df..cdfb7d42b72b 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -135,7 +135,6 @@ envoy_benchmark_test( envoy_cc_test( name = "health_checker_impl_test", srcs = ["health_checker_impl_test.cc"], - tags = ["fails_on_windows"], deps = [ ":utility_lib", "//source/common/buffer:buffer_lib", @@ -405,7 +404,6 @@ envoy_benchmark_test( name = "load_balancer_benchmark_test", timeout = "long", benchmark_binary = "load_balancer_benchmark", - tags = ["fails_on_windows"], ) envoy_cc_test( diff --git a/test/extensions/filters/network/rocketmq_proxy/BUILD b/test/extensions/filters/network/rocketmq_proxy/BUILD index f01055ab6742..82a70612767f 100644 --- a/test/extensions/filters/network/rocketmq_proxy/BUILD +++ b/test/extensions/filters/network/rocketmq_proxy/BUILD @@ -47,7 +47,6 @@ envoy_extension_cc_test( name = "router_test", srcs = ["router_test.cc"], extension_name = "envoy.filters.network.rocketmq_proxy", - tags = ["fails_on_windows"], deps = [ ":mocks_lib", ":utility_lib", diff --git a/test/server/BUILD b/test/server/BUILD index 84a3d3fb8899..4718ce6669fb 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -355,7 +355,6 @@ envoy_cc_test( ":server_test_data", ":static_validation_test_data", ], - tags = ["fails_on_windows"], deps = [ "//source/common/version:version_lib", "//source/extensions/access_loggers/file:config", From d651d2e048a7ddc977c028755287b545dd606adf Mon Sep 17 00:00:00 2001 From: jianwen612 <55008549+jianwen612@users.noreply.github.com> Date: Mon, 10 Aug 2020 16:15:30 -0500 Subject: [PATCH 20/67] [fuzz]network-level WriteFilter generic fuzzer (#12462) * added generic freamework for testing filters. Signed-off-by: jianwen * added code for covering ext_authz filter Signed-off-by: jianwen * restore the log output in ext_authz implementation. Signed-off-by: jianwen * fixed style problem Signed-off-by: jianwen * fixed style problem Signed-off-by: jianwen * added comments Signed-off-by: jianwen * added ststem time control for local_rate_limit Signed-off-by: jianwen * enabled three filters coverage Signed-off-by: jianwen * added support for ext_authz response Signed-off-by: jianwen * added coverage for tcp_proxy and client_ssl_auth. Increased the coverage for ext_auth by enabling the mocked response. Fixed the validation problem inside client_ssl_auth's protobuf Signed-off-by: jianwen * removed test for tcp_proxy filter Signed-off-by: jianwen * fix bazel style Signed-off-by: jianwen * fixed style Signed-off-by: jianwen * found issues in tcp_proxy and direct_response. added test cases for the issues Signed-off-by: jianwen * replace raw string names with names from factory Signed-off-by: jianwen * added test cases for direct response and sni_cluster Signed-off-by: jianwen * cleaned the code Signed-off-by: jianwen * deleted some useless comments Signed-off-by: jianwen * removed filters with known issues from the fuzzer Signed-off-by: jianwen * removed unnecessary corpus Signed-off-by: jianwen * fix the style Signed-off-by: jianwen * removed unsupported test cases Signed-off-by: jianwen * removed unnecessary comments Signed-off-by: jianwen * removed the empty destructor of fakeFactoryContext Signed-off-by: jianwen * fixed naming problems and removed the constructor of fake class Signed-off-by: jianwen * start working on http_connection_manager and solved one potential use-after-free problem. Signed-off-by: jianwen * fixed style problems Signed-off-by: jianwen * modified ON_CALL to EXPECT_CALL.WillOnce for some unique_ptr. Removed ON_CALL for addr_, instead, directly change the pointer inside connection_ Signed-off-by: jianwen * run fix code style Signed-off-by: jianwen * added HCM filter and SDFP filter Signed-off-by: jianwen * fixed typos and added TODOs Signed-off-by: jianwen * fixed a typo Signed-off-by: jianwen * separate the fake class definition and the per_filter processing in different files. Cleaned up the deps Signed-off-by: jianwen * added comments and assert() Signed-off-by: jianwen * fix style Signed-off-by: jianwen * fixed the proto definition on ThriftProxy.Route.RouteAction.cluter_header. Signed-off-by: jianwen * run proto fix after modification in route.proto Signed-off-by: jianwen * added comment on seconds_in_one_day_ Signed-off-by: jianwen * added test cases Signed-off-by: jianwen * trying to add valid filters Signed-off-by: jianwen * added test case for thrift proxy and added deps Signed-off-by: jianwen * added comment and log Signed-off-by: jianwen * refined the test cases Signed-off-by: jianwen * added dict Signed-off-by: jianwen * added dict to BUILD Signed-off-by: jianwen * added support for rocketmq_proxy Signed-off-by: jianwen * refined test cases for kafka and rocketmq Signed-off-by: jianwen * added support for RateLimit and a test case for it. Signed-off-by: jianwen * renamed the filter fuzzer to readfilter fuzzer Signed-off-by: jianwen * fixed code style Signed-off-by: jianwen * merged generic fuzzer(rename) Signed-off-by: jianwen * added rbac Signed-off-by: jianwen * fix nits Signed-off-by: jianwen * fixed style Signed-off-by: jianwen * fix style Signed-off-by: jianwen * removed several test cases Signed-off-by: jianwen * removed several test cases Signed-off-by: jianwen * adde comments Signed-off-by: jianwen * added writefilter fuzzer and a crash testcase for zookeeperproxy Signed-off-by: jianwen * covered all the filters Signed-off-by: jianwen * added a comment for postgres_proxy Signed-off-by: jianwen * fixed style Signed-off-by: jianwen * fixed TODO name Signed-off-by: jianwen * removed unrelevant changes Signed-off-by: jianwen * fixed proto Signed-off-by: jianwen * restore the changes Signed-off-by: jianwen * trying to add coverage for mongodb Signed-off-by: jianwen * fixed style and removed cout Signed-off-by: jianwen * added time source Signed-off-by: jianwen * added a test case for mongo, fixed style problem Signed-off-by: jianwen * fixed a spelling problem Signed-off-by: jianwen * removed the test case for postgres_proxy Signed-off-by: jianwen * added new lines and made the comment clearer Signed-off-by: jianwen * fixed a spelling problem Signed-off-by: jianwen * removed a hardcoded debug code, used debug log in writefilter fuzzer. Signed-off-by: jianwen * removed unnecessary BUILD deps. Signed-off-by: jianwen --- .../filters/network/common/fuzz/BUILD | 44 +++++++ .../network_writefilter_corpus/kafka_broker_1 | 110 ++++++++++++++++ .../mongodb_proxy_1 | 107 +++++++++++++++ .../network_writefilter_corpus/mysql_proxy_1 | 86 ++++++++++++ .../zookeeper_proxy_1 | 17 +++ .../zookeeper_proxy_assert_failure_onwrite | 12 ++ .../fuzz/network_writefilter_fuzz.proto | 31 +++++ .../fuzz/network_writefilter_fuzz_test.cc | 58 +++++++++ .../common/fuzz/uber_per_writefilter.cc | 35 +++++ .../network/common/fuzz/uber_writefilter.cc | 123 ++++++++++++++++++ .../network/common/fuzz/uber_writefilter.h | 40 ++++++ 11 files changed, 663 insertions(+) create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_1 create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_1 create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_1 create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_1 create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_assert_failure_onwrite create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_fuzz.proto create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc create mode 100644 test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc create mode 100644 test/extensions/filters/network/common/fuzz/uber_writefilter.cc create mode 100644 test/extensions/filters/network/common/fuzz/uber_writefilter.h diff --git a/test/extensions/filters/network/common/fuzz/BUILD b/test/extensions/filters/network/common/fuzz/BUILD index f8d38307d569..8f54f57e5de8 100644 --- a/test/extensions/filters/network/common/fuzz/BUILD +++ b/test/extensions/filters/network/common/fuzz/BUILD @@ -23,6 +23,15 @@ envoy_proto_library( ], ) +envoy_proto_library( + name = "network_writefilter_fuzz_proto", + srcs = ["network_writefilter_fuzz.proto"], + deps = [ + "//test/fuzz:common_proto", + "@envoy_api//envoy/config/listener/v3:pkg", + ], +) + envoy_cc_test_library( name = "uber_readfilter_lib", srcs = [ @@ -59,3 +68,38 @@ envoy_cc_fuzz_test( "//test/config:utility_lib", ] + envoy_all_network_filters(), ) + +envoy_cc_test_library( + name = "uber_writefilter_lib", + srcs = [ + "uber_per_writefilter.cc", + "uber_writefilter.cc", + ], + hdrs = ["uber_writefilter.h"], + deps = [ + ":network_writefilter_fuzz_proto_cc_proto", + "//source/common/config:utility_lib", + "//source/extensions/filters/network:well_known_names", + "//source/extensions/filters/network/common:utility_lib", + "//test/extensions/filters/network/common/fuzz/utils:network_filter_fuzzer_fakes_lib", + "//test/fuzz:utility_lib", + "//test/mocks/network:network_mocks", + ], +) + +envoy_cc_fuzz_test( + name = "network_writefilter_fuzz_test", + srcs = ["network_writefilter_fuzz_test.cc"], + corpus = "network_writefilter_corpus", + # All Envoy network filters must be linked to the test in order for the fuzzer to pick + # these up via the NamedNetworkFilterConfigFactory. + deps = [ + ":uber_writefilter_lib", + "//source/common/config:utility_lib", + "//source/extensions/filters/network/kafka:kafka_broker_config_lib", + "//source/extensions/filters/network/mongo_proxy:config", + "//source/extensions/filters/network/mysql_proxy:config", + "//source/extensions/filters/network/zookeeper_proxy:config", + "//test/config:utility_lib", + ], +) diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_1 b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_1 new file mode 100644 index 000000000000..a20c58dd2d4a --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_1 @@ -0,0 +1,110 @@ +config { + name: "envoy.filters.network.kafka_broker" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker" + value: "\n}\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } +} +actions { + on_write { + data: "-" + end_stream: true + } +} +actions { + advance_time { + milliseconds: 268435 + } +} +actions { + on_write { + data: "-" + end_stream: true + } +} +actions { + on_write { + data: "\312\312\312\312\312\312\312\312\312\312\312\312\315\312\312\312\312\312\312\312\312\312\312" + end_stream: true + } +} +actions { + on_write { + data: "-" + } +} +actions { + on_write { + data: "\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312\312" + end_stream: true + } +} +actions { + on_write { + data: "-" + end_stream: true + } +} +actions { + on_write { + data: "-" + end_stream: true + } +} +actions { + on_write { + data: "-" + } +} +actions { + on_write { + data: "-" + } +} +actions { + on_write { + data: "\n\002\315\265" + } +} +actions { + on_write { + end_stream: true + } +} +actions { + on_write { + data: "\020\000\000\000" + } +} +actions { + on_write { + data: "-" + end_stream: true + } +} +actions { + on_write { + data: "-" + end_stream: true + } +} +actions { + on_write { + data: "p" + } +} +actions { + on_write { + data: "-" + } +} +actions { + on_write { + data: "-" + end_stream: true + } +} +actions { + on_write { + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_1 b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_1 new file mode 100644 index 000000000000..20a344f8fe35 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_1 @@ -0,0 +1,107 @@ +config { + name: "envoy.filters.network.mongo_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.mongo_proxy.v3.MongoProxy" + value: "\n\001\\\032\007\"\003\010\200t*\000 \001" + } +} +actions { + on_write { + data: "]\000" + } +} +actions { + on_write { + data: "\004\000" + end_stream: true + } +} +actions { + advance_time { + milliseconds: 14848 + } +} +actions { + on_write { + data: "\004\000" + end_stream: true + } +} +actions { + advance_time { + milliseconds: 14848 + } +} +actions { + on_write { + data: "\004\000\001\000\000\000\000\000\000\001" + end_stream: true + } +} +actions { + on_write { + data: "<" + end_stream: true + } +} +actions { + on_write { + data: "\004\000" + } +} +actions { + on_write { + data: "\004\000" + } +} +actions { + advance_time { + milliseconds: 14848 + } +} +actions { + on_write { + data: "type.googleapis.com/envoy.extensions.filters.network.mongo_proxy.v3.MongoProxy" + end_stream: true + } +} +actions { + on_write { + data: "\004\000" + end_stream: true + } +} +actions { + on_write { + data: "\004\000" + } +} +actions { + on_write { + data: "pH\037\000 `\000\000" + end_stream: true + } +} +actions { + on_write { + data: "\004\000" + end_stream: true + } +} +actions { + advance_time { + milliseconds: 14848 + } +} +actions { + on_write { + data: "=" + end_stream: true + } +} +actions { + on_write { + data: "\004\000" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_1 b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_1 new file mode 100644 index 000000000000..f58ad110b8b9 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_1 @@ -0,0 +1,86 @@ +config { + name: "envoy.filters.network.mysql_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.mysql_proxy.v3.MySQLProxy" + value: "\n\006#\336\215\302\246\001" + } +} +actions { + on_write { + data: "\031\031\031\031" + } +} +actions { + on_write { + data: "\031\031\031\031\031\031\031\031" + end_stream: true + } +} +actions { + on_write { + data: "3" + } +} +actions { + on_write { + data: "#" + } +} +actions { + on_write { + data: "#" + end_stream: true + } +} +actions { + on_write { + data: "3" + } +} +actions { + on_write { + data: "#" + end_stream: true + } +} +actions { + on_write { + data: "#" + } +} +actions { + on_write { + data: "#" + } +} +actions { + on_write { + data: "\031\031\031\031\031\031\031\031" + end_stream: true + } +} +actions { + on_write { + end_stream: true + } +} +actions { + on_write { + end_stream: true + } +} +actions { + on_write { + data: "3" + } +} +actions { + on_write { + end_stream: true + } +} +actions { + on_write { + data: "3" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_1 b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_1 new file mode 100644 index 000000000000..2e2e6c1bfb8d --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_1 @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\nVtype.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy\032\000" + } +} +actions { + on_write { + data: "\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030c.googlers.com\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030\030" + } +} +actions { + on_write { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_assert_failure_onwrite b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_assert_failure_onwrite new file mode 100644 index 000000000000..ae270c6fe26c --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_assert_failure_onwrite @@ -0,0 +1,12 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\nVtype.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy\022\001!\032\006\010\377\376\377\317\017" + } +} +actions { + on_write { + data: "\030\030\030\030\030\030\030\030" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz.proto b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz.proto new file mode 100644 index 000000000000..77de32b5858f --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package test.extensions.filters.network; +import "validate/validate.proto"; +import "envoy/config/listener/v3/listener_components.proto"; + +message OnWrite { + bytes data = 1; + bool end_stream = 2; +} + +message AdvanceTime { + // Advance the system time by (0,24] hours. + uint32 milliseconds = 1 [(validate.rules).uint32 = {gt: 0 lt: 86400000}]; +} + +message WriteAction { + oneof action_selector { + option (validate.required) = true; + // Call onWrite() + OnWrite on_write = 2; + // Advance time_source_ + AdvanceTime advance_time = 3; + } +} + +message FilterFuzzTestCase { + // This is actually a protobuf type for the config of network filters. + envoy.config.listener.v3.Filter config = 1; + repeated WriteAction actions = 2; +} diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc new file mode 100644 index 000000000000..702cb4078db4 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_fuzz_test.cc @@ -0,0 +1,58 @@ +#include "common/config/utility.h" +#include "common/protobuf/utility.h" + +#include "extensions/filters/network/well_known_names.h" + +#include "test/config/utility.h" +#include "test/extensions/filters/network/common/fuzz/network_writefilter_fuzz.pb.validate.h" +#include "test/extensions/filters/network/common/fuzz/uber_writefilter.h" +#include "test/fuzz/fuzz_runner.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +DEFINE_PROTO_FUZZER(const test::extensions::filters::network::FilterFuzzTestCase& input) { + ABSL_ATTRIBUTE_UNUSED static PostProcessorRegistration reg = { + [](test::extensions::filters::network::FilterFuzzTestCase* input, unsigned int seed) { + // This post-processor mutation is applied only when libprotobuf-mutator + // calls mutate on an input, and *not* during fuzz target execution. + // Replaying a corpus through the fuzzer will not be affected by the + // post-processor mutation. + + // TODO(jianwendong): consider using a factory to store the names of all + // writeFilters. + static const auto filter_names = UberWriteFilterFuzzer::filterNames(); + static const auto factories = Registry::FactoryRegistry< + Server::Configuration::NamedNetworkFilterConfigFactory>::factories(); + // Choose a valid filter name. + if (std::find(filter_names.begin(), filter_names.end(), input->config().name()) == + std::end(filter_names)) { + absl::string_view filter_name = filter_names[seed % filter_names.size()]; + input->mutable_config()->set_name(std::string(filter_name)); + } + // Set the corresponding type_url for Any. + auto& factory = factories.at(input->config().name()); + input->mutable_config()->mutable_typed_config()->set_type_url( + absl::StrCat("type.googleapis.com/", + factory->createEmptyConfigProto()->GetDescriptor()->full_name())); + }}; + try { + TestUtility::validate(input); + // Check the filter's name in case some filters are not supported yet. + // TODO(jianwendong): remove this if block when we have a factory for writeFilters. + static const auto filter_names = UberWriteFilterFuzzer::filterNames(); + if (std::find(filter_names.begin(), filter_names.end(), input.config().name()) == + std::end(filter_names)) { + ENVOY_LOG_MISC(debug, "Test case with unsupported filter type: {}", input.config().name()); + return; + } + static UberWriteFilterFuzzer fuzzer; + fuzzer.fuzz(input.config(), input.actions()); + } catch (const ProtoValidationException& e) { + ENVOY_LOG_MISC(debug, "ProtoValidationException: {}", e.what()); + } +} + +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc b/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc new file mode 100644 index 000000000000..911caa250c52 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/uber_per_writefilter.cc @@ -0,0 +1,35 @@ +#include "extensions/filters/network/common/utility.h" +#include "extensions/filters/network/well_known_names.h" + +#include "test/extensions/filters/network/common/fuzz/uber_writefilter.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +std::vector UberWriteFilterFuzzer::filterNames() { + // These filters have already been covered by this fuzzer. + // Will extend to cover other network filters one by one. + static std::vector filter_names; + if (filter_names.empty()) { + const auto factories = Registry::FactoryRegistry< + Server::Configuration::NamedNetworkFilterConfigFactory>::factories(); + const std::vector supported_filter_names = { + NetworkFilterNames::get().ZooKeeperProxy, NetworkFilterNames::get().KafkaBroker, + NetworkFilterNames::get().MongoProxy, NetworkFilterNames::get().MySQLProxy + // TODO(jianwendong) Add "NetworkFilterNames::get().Postgres" after it supports untrusted + // data. + }; + for (auto& filter_name : supported_filter_names) { + if (factories.contains(filter_name)) { + filter_names.push_back(filter_name); + } else { + ENVOY_LOG_MISC(debug, "Filter name not found in the factory: {}", filter_name); + } + } + } + return filter_names; +} + +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/common/fuzz/uber_writefilter.cc b/test/extensions/filters/network/common/fuzz/uber_writefilter.cc new file mode 100644 index 000000000000..517429a1dd4b --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/uber_writefilter.cc @@ -0,0 +1,123 @@ +#include "test/extensions/filters/network/common/fuzz/uber_writefilter.h" + +#include "common/config/utility.h" +#include "common/config/version_converter.h" + +using testing::_; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +void UberWriteFilterFuzzer::reset() { + // Reset the state of dependencies so that a new fuzz input starts in a clean state. + + // Close the connection to make sure the filter's callback is set to nullptr. + write_filter_callbacks_->connection_.raiseEvent(Network::ConnectionEvent::LocalClose); + // Clear the filter's raw pointer stored inside the connection_ and reset the connection_'s state. + write_filter_callbacks_->connection_.callbacks_.clear(); + write_filter_callbacks_->connection_.bytes_sent_callbacks_.clear(); + write_filter_callbacks_->connection_.state_ = Network::Connection::State::Open; + // Clear the pointers inside the mock_dispatcher + Event::MockDispatcher& mock_dispatcher = + dynamic_cast(write_filter_callbacks_->connection_.dispatcher_); + mock_dispatcher.clearDeferredDeleteList(); + write_filter_.reset(); +} + +void UberWriteFilterFuzzer::fuzzerSetup() { + // Setup process when this fuzzer object is constructed. + // For a static fuzzer, this will only be executed once. + + // Get the pointer of write_filter when the write_filter is being added to connection_. + write_filter_callbacks_ = std::make_shared>(); + read_filter_callbacks_ = std::make_shared>(); + ON_CALL(write_filter_callbacks_->connection_, addWriteFilter(_)) + .WillByDefault(Invoke([&](Network::WriteFilterSharedPtr write_filter) -> void { + write_filter->initializeWriteFilterCallbacks(*write_filter_callbacks_); + write_filter_ = write_filter; + })); + ON_CALL(write_filter_callbacks_->connection_, addFilter(_)) + .WillByDefault(Invoke([&](Network::FilterSharedPtr filter) -> void { + filter->initializeReadFilterCallbacks(*read_filter_callbacks_); + filter->initializeWriteFilterCallbacks(*write_filter_callbacks_); + write_filter_ = filter; + })); + factory_context_.prepareSimulatedSystemTime(); + + // Set featureEnabled for mongo_proxy + ON_CALL(factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.proxy_enabled", 100)) + .WillByDefault(Return(true)); + ON_CALL(factory_context_.runtime_loader_.snapshot_, + featureEnabled("mongo.connection_logging_enabled", 100)) + .WillByDefault(Return(true)); + ON_CALL(factory_context_.runtime_loader_.snapshot_, featureEnabled("mongo.logging_enabled", 100)) + .WillByDefault(Return(true)); + + // Set featureEnabled for thrift_proxy + ON_CALL(factory_context_.runtime_loader_.snapshot_, + featureEnabled("ratelimit.thrift_filter_enabled", 100)) + .WillByDefault(Return(true)); + ON_CALL(factory_context_.runtime_loader_.snapshot_, + featureEnabled("ratelimit.thrift_filter_enforcing", 100)) + .WillByDefault(Return(true)); + ON_CALL(factory_context_.runtime_loader_.snapshot_, + featureEnabled("ratelimit.test_key.thrift_filter_enabled", 100)) + .WillByDefault(Return(true)); +} + +UberWriteFilterFuzzer::UberWriteFilterFuzzer() + : time_source_(factory_context_.simulatedTimeSystem()) { + fuzzerSetup(); +} + +void UberWriteFilterFuzzer::fuzz( + const envoy::config::listener::v3::Filter& proto_config, + const Protobuf::RepeatedPtrField<::test::extensions::filters::network::WriteAction>& actions) { + try { + // Try to create the filter callback(cb_). Exit early if the config is invalid or violates PGV + // constraints. + const std::string& filter_name = proto_config.name(); + ENVOY_LOG_MISC(debug, "filter name {}", filter_name); + auto& factory = Config::Utility::getAndCheckFactoryByName< + Server::Configuration::NamedNetworkFilterConfigFactory>(filter_name); + ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( + proto_config, factory_context_.messageValidationVisitor(), factory); + ENVOY_LOG_MISC(debug, "Config content after decoded: {}", message->DebugString()); + cb_ = factory.createFilterFactoryFromProto(*message, factory_context_); + // Add filter to connection_. + cb_(write_filter_callbacks_->connection_); + } catch (const EnvoyException& e) { + ENVOY_LOG_MISC(debug, "Controlled exception in filter setup {}", e.what()); + return; + } + for (const auto& action : actions) { + ENVOY_LOG_MISC(debug, "action {}", action.DebugString()); + switch (action.action_selector_case()) { + case test::extensions::filters::network::WriteAction::kOnWrite: { + ASSERT(write_filter_ != nullptr); + Buffer::OwnedImpl buffer(action.on_write().data()); + write_filter_->onWrite(buffer, action.on_write().end_stream()); + + break; + } + case test::extensions::filters::network::WriteAction::kAdvanceTime: { + time_source_.advanceTimeAsync( + std::chrono::milliseconds(action.advance_time().milliseconds())); + factory_context_.dispatcher().run(Event::Dispatcher::RunType::NonBlock); + break; + } + default: { + // Unhandled actions. + ENVOY_LOG_MISC(debug, "Action support is missing for:\n{}", action.DebugString()); + PANIC("A case is missing for an action"); + } + } + } + + reset(); +} + +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/common/fuzz/uber_writefilter.h b/test/extensions/filters/network/common/fuzz/uber_writefilter.h new file mode 100644 index 000000000000..9f6c34eb60e9 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/uber_writefilter.h @@ -0,0 +1,40 @@ +#include "envoy/network/filter.h" + +#include "common/protobuf/protobuf.h" + +#include "test/extensions/filters/network/common/fuzz/network_writefilter_fuzz.pb.validate.h" +#include "test/extensions/filters/network/common/fuzz/utils/fakes.h" +#include "test/mocks/network/mocks.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { + +class UberWriteFilterFuzzer { +public: + UberWriteFilterFuzzer(); + // This creates the filter config and runs the fuzzed data against the filter. + void fuzz( + const envoy::config::listener::v3::Filter& proto_config, + const Protobuf::RepeatedPtrField<::test::extensions::filters::network::WriteAction>& actions); + // Get the name of filters which has been covered by this fuzzer. + static std::vector filterNames(); + +protected: + // Set-up filter specific mock expectations in constructor. + void fuzzerSetup(); + // Reset the states of the mock objects. + void reset(); + +private: + Server::Configuration::FakeFactoryContext factory_context_; + Event::SimulatedTimeSystem& time_source_; + Network::WriteFilterSharedPtr write_filter_; + Network::FilterFactoryCb cb_; + std::shared_ptr> write_filter_callbacks_; + std::shared_ptr> read_filter_callbacks_; +}; + +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy From 3cb95f6ff88c7e9681f77fa12df386c3e10f6e7f Mon Sep 17 00:00:00 2001 From: Yifan Yang Date: Mon, 10 Aug 2020 17:23:52 -0400 Subject: [PATCH 21/67] Cleanup: cleaning up .first/.second [source/common/config] (#12468) This is the second PR to this #12354. This time is in the common/config submodule. Signed-off-by: Yifan Yang --- source/common/config/delta_subscription_state.cc | 8 ++++---- source/common/config/metadata.h | 7 ++++--- source/common/config/new_grpc_mux_impl.cc | 8 ++++---- source/common/config/type_to_endpoint.cc | 8 +++----- source/common/config/watch_map.cc | 13 ++++++------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 2763fdfd9dff..c0a6a5502cb0 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -124,16 +124,16 @@ DeltaSubscriptionState::getNextRequestAckless() { // initial_resource_versions "must be populated for first request in a stream". // Also, since this might be a new server, we must explicitly state *all* of our subscription // interest. - for (auto const& resource : resource_versions_) { + for (auto const& [resource_name, resource_version] : resource_versions_) { // Populate initial_resource_versions with the resource versions we currently have. // Resources we are interested in, but are still waiting to get any version of from the // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) - if (!resource.second.waitingForServer()) { - (*request.mutable_initial_resource_versions())[resource.first] = resource.second.version(); + if (!resource_version.waitingForServer()) { + (*request.mutable_initial_resource_versions())[resource_name] = resource_version.version(); } // As mentioned above, fill resource_names_subscribe with everything, including names we // have yet to receive any resource for. - names_added_.insert(resource.first); + names_added_.insert(resource_name); } names_removed_.clear(); } diff --git a/source/common/config/metadata.h b/source/common/config/metadata.h index ed4fcd96c270..efac4eff7e59 100644 --- a/source/common/config/metadata.h +++ b/source/common/config/metadata.h @@ -116,10 +116,11 @@ template class TypedMetadataImpl : public TypedMetadata */ void populateFrom(const envoy::config::core::v3::Metadata& metadata) { auto& data_by_key = metadata.filter_metadata(); - for (const auto& it : Registry::FactoryRegistry::factories()) { - const auto& meta_iter = data_by_key.find(it.first); + for (const auto& [factory_name, factory] : + Registry::FactoryRegistry::factories()) { + const auto& meta_iter = data_by_key.find(factory_name); if (meta_iter != data_by_key.end()) { - data_[it.second->name()] = it.second->parse(meta_iter->second); + data_[factory->name()] = factory->parse(meta_iter->second); } } } diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index c7caaf04f664..131ccd24db51 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -73,8 +73,8 @@ void NewGrpcMuxImpl::onDiscoveryResponse( } void NewGrpcMuxImpl::onStreamEstablished() { - for (auto& sub : subscriptions_) { - sub.second->sub_state_.markStreamFresh(); + for (auto& [type_url, subscription] : subscriptions_) { + subscription->sub_state_.markStreamFresh(); } trySendDiscoveryRequests(); } @@ -88,8 +88,8 @@ void NewGrpcMuxImpl::onEstablishmentFailure() { absl::flat_hash_map all_subscribed; absl::flat_hash_map already_called; do { - for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = &sub.second->sub_state_; + for (auto& [type_url, subscription] : subscriptions_) { + all_subscribed[type_url] = &subscription->sub_state_; } for (auto& sub : all_subscribed) { if (already_called.insert(sub).second) { // insert succeeded ==> not already called diff --git a/source/common/config/type_to_endpoint.cc b/source/common/config/type_to_endpoint.cc index 9821b288dcbc..1c32fe47ad2c 100644 --- a/source/common/config/type_to_endpoint.cc +++ b/source/common/config/type_to_endpoint.cc @@ -185,12 +185,11 @@ TypeUrlToVersionedServiceMap* buildTypeUrlToServiceMap() { }}, }}, }) { - for (const auto& registered_service : registered) { - const TypeUrl resource_type_url = getResourceTypeUrl(registered_service.first); + for (const auto& [registered_service_name, registered_service_info] : registered) { + const TypeUrl resource_type_url = getResourceTypeUrl(registered_service_name); VersionedService& service = (*type_url_to_versioned_service_map)[resource_type_url]; - for (const auto& versioned_service_name : registered_service.second.names_) { - const ServiceName& service_name = versioned_service_name.second; + for (const auto& [transport_api_version, service_name] : registered_service_info.names_) { const auto* service_desc = Protobuf::DescriptorPool::generated_pool()->FindServiceByName(service_name); ASSERT(service_desc != nullptr, fmt::format("{} missing", service_name)); @@ -200,7 +199,6 @@ TypeUrlToVersionedServiceMap* buildTypeUrlToServiceMap() { // services don't implement all, e.g. VHDS doesn't support SotW or REST. for (int method_index = 0; method_index < service_desc->method_count(); ++method_index) { const auto& method_desc = *service_desc->method(method_index); - const auto transport_api_version = versioned_service_name.first; if (absl::StartsWith(method_desc.name(), "Stream")) { service.sotw_grpc_.methods_[transport_api_version] = method_desc.full_name(); } else if (absl::StartsWith(method_desc.name(), "Delta")) { diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index f17d01decbc4..51e73e06344d 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -181,28 +181,27 @@ void WatchMap::onConfigUpdate( } // We just bundled up the updates into nice per-watch packages. Now, deliver them. - for (const auto& added : per_watch_added) { - const Watch* cur_watch = added.first; + for (const auto& [cur_watch, resource_to_add] : per_watch_added) { if (deferred_removed_during_update_->count(cur_watch) > 0) { continue; } const auto removed = per_watch_removed.find(cur_watch); if (removed == per_watch_removed.end()) { // additions only, no removals - cur_watch->callbacks_.onConfigUpdate(added.second, {}, system_version_info); + cur_watch->callbacks_.onConfigUpdate(resource_to_add, {}, system_version_info); } else { // both additions and removals - cur_watch->callbacks_.onConfigUpdate(added.second, removed->second, system_version_info); + cur_watch->callbacks_.onConfigUpdate(resource_to_add, removed->second, system_version_info); // Drop the removals now, so the final removals-only pass won't use them. per_watch_removed.erase(removed); } } // Any removals-only updates will not have been picked up in the per_watch_added loop. - for (auto& removed : per_watch_removed) { - if (deferred_removed_during_update_->count(removed.first) > 0) { + for (auto& [cur_watch, resource_to_remove] : per_watch_removed) { + if (deferred_removed_during_update_->count(cur_watch) > 0) { continue; } - removed.first->callbacks_.onConfigUpdate({}, removed.second, system_version_info); + cur_watch->callbacks_.onConfigUpdate({}, resource_to_remove, system_version_info); } } From 5200978ba69488fde6930d6059009dacc56c1772 Mon Sep 17 00:00:00 2001 From: justin-mp Date: Mon, 10 Aug 2020 17:24:38 -0400 Subject: [PATCH 22/67] Add const qualifier to returned pointers in MockIp and MockResolvedAddress (#12573) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compared to their superclasses, the MockIp and MockResolvedAddress classes dropp the const qualifier from the pointers they return. Since the non-mock class implementing these methods return const pointers, it’s difficult to use non-mock classes with these mocks without resorting to hacks like const_cast. Signed-off-by: Justin Mazzola Paluska --- test/mocks/network/mocks.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 596afe6ffbea..45371be5d584 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -413,8 +413,8 @@ class MockIp : public Address::Ip { MOCK_METHOD(const std::string&, addressAsString, (), (const)); MOCK_METHOD(bool, isAnyAddress, (), (const)); MOCK_METHOD(bool, isUnicastAddress, (), (const)); - MOCK_METHOD(Address::Ipv4*, ipv4, (), (const)); - MOCK_METHOD(Address::Ipv6*, ipv6, (), (const)); + MOCK_METHOD(const Address::Ipv4*, ipv4, (), (const)); + MOCK_METHOD(const Address::Ipv6*, ipv6, (), (const)); MOCK_METHOD(uint32_t, port, (), (const)); MOCK_METHOD(Address::IpVersion, version, (), (const)); MOCK_METHOD(bool, v6only, (), (const)); @@ -432,11 +432,11 @@ class MockResolvedAddress : public Address::Instance { MOCK_METHOD(Api::SysCallIntResult, bind, (os_fd_t), (const)); MOCK_METHOD(Api::SysCallIntResult, connect, (os_fd_t), (const)); - MOCK_METHOD(Address::Ip*, ip, (), (const)); - MOCK_METHOD(Address::Pipe*, pipe, (), (const)); + MOCK_METHOD(const Address::Ip*, ip, (), (const)); + MOCK_METHOD(const Address::Pipe*, pipe, (), (const)); MOCK_METHOD(IoHandlePtr, socket, (Socket::Type), (const)); MOCK_METHOD(Address::Type, type, (), (const)); - MOCK_METHOD(sockaddr*, sockAddr, (), (const)); + MOCK_METHOD(const sockaddr*, sockAddr, (), (const)); MOCK_METHOD(socklen_t, sockAddrLen, (), (const)); const std::string& asString() const override { return physical_; } From cce07c0396190915ab3d72a9e0a6d1c4ba5aa1a2 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Mon, 10 Aug 2020 16:28:30 -0700 Subject: [PATCH 23/67] abseil: update to latest (#12561) Pulls in TSAN mutex fixes as well as works around false detection. Signed-off-by: Matt Klein --- .bazelrc | 3 +++ bazel/repository_locations.bzl | 8 ++++---- test/integration/fake_upstream.cc | 2 +- test/integration/fake_upstream.h | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.bazelrc b/.bazelrc index 1cb3caaa6a9f..fa8d80a0242d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -89,6 +89,9 @@ build:clang-tsan --build_tag_filters=-no_san,-no_tsan build:clang-tsan --test_tag_filters=-no_san,-no_tsan # Needed due to https://github.com/libevent/libevent/issues/777 build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE +# https://github.com/abseil/abseil-cpp/issues/760 +# https://github.com/google/sanitizers/issues/953 +build:clang-tsan --test_env="TSAN_OPTIONS=report_atomic_races=0" # Clang MSAN - this is the base config for remote-msan and docker-msan. To run this config without # our build image, follow https://github.com/google/sanitizers/wiki/MemorySanitizerLibcxxHowTo diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 73698991a2f4..11cf908a21c0 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -95,10 +95,10 @@ DEPENDENCY_REPOSITORIES = dict( cpe = "N/A", ), com_google_absl = dict( - sha256 = "ec8ef47335310cc3382bdc0d0cc1097a001e67dc83fcba807845aa5696e7e1e4", - strip_prefix = "abseil-cpp-302b250e1d917ede77b5ff00a6fd9f28430f1563", - # 2020-07-13 - urls = ["https://github.com/abseil/abseil-cpp/archive/302b250e1d917ede77b5ff00a6fd9f28430f1563.tar.gz"], + sha256 = "573baccd67aa591b8c7209bfb0c77e0d15633d77ced39d1ccbb1232828f7f7d9", + strip_prefix = "abseil-cpp-ce4bc927755fdf0ed03d679d9c7fa041175bb3cb", + # 2020-08-08 + urls = ["https://github.com/abseil/abseil-cpp/archive/ce4bc927755fdf0ed03d679d9c7fa041175bb3cb.tar.gz"], use_category = ["dataplane", "controlplane"], cpe = "N/A", ), diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 476922bd8a68..b20ff0318398 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -709,7 +709,7 @@ FakeRawConnection::waitForData(const std::function& da AssertionResult FakeRawConnection::write(const std::string& data, bool end_stream, milliseconds timeout) { return shared_connection_.executeOnDispatcher( - [&data, end_stream](Network::Connection& connection) { + [data, end_stream](Network::Connection& connection) { Buffer::OwnedImpl to_write(data); connection.write(to_write, end_stream); }, diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 46287d7a5191..0c23c42e4b41 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -300,7 +300,7 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks { return testing::AssertionSuccess(); } Thread::CondVar callback_ready_event; - bool unexpected_disconnect = false; + std::atomic unexpected_disconnect = false; connection_.dispatcher().post( [this, f, &callback_ready_event, &unexpected_disconnect]() -> void { // The use of connected() here, vs. !disconnected_, is because we want to use the lock_ From 04de1cfb2bb7a774dd38772c7e262af3e61727b3 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Mon, 10 Aug 2020 20:01:24 -0400 Subject: [PATCH 24/67] stats: keep a set of ref-counted parent histograms in ThreadLocalStore so that two with the same name map to the same histogram object. (#12275) Commit Message: Creates a storage model for thread-local histograms enabling continuity of data across scope reloads. Previously, whenever a Scope was re-created, the counters, gauges, and text-readouts of the same names would retain their previous values. However, fresh histograms were created on every scope reload, and stats dumps would include duplicate histograms with the same name. This change adds an analogous name-based set of histograms, held in ThreadLocalStore, so that we have a single histogram representing all its generations of data. This is somewhat more complex than for the other stats, since there were thread-local buffers, which previously were owned by TlsScope and needed to be made independent. So this introduces a new tls histogram map in the tls-cache to maintain this. This should help unblock #12241. Additional Description: Risk Level: high (not clear whether this is enough testing of histogram usage) Testing: //test/... Docs Changes: n/a Release Notes: n/a Fixes: #3098 Signed-off-by: Joshua Marantz --- source/common/stats/allocator_impl.h | 19 -- source/common/stats/metric_impl.h | 25 ++ source/common/stats/thread_local_store.cc | 188 +++++++++--- source/common/stats/thread_local_store.h | 83 ++--- source/docs/stats.md | 14 +- source/server/admin/stats_handler.cc | 8 +- test/common/stats/thread_local_store_test.cc | 306 ++++++++++++++++--- test/integration/stats_integration_test.cc | 8 +- 8 files changed, 504 insertions(+), 147 deletions(-) diff --git a/source/common/stats/allocator_impl.h b/source/common/stats/allocator_impl.h index 3242d0de5fef..469484866f18 100644 --- a/source/common/stats/allocator_impl.h +++ b/source/common/stats/allocator_impl.h @@ -58,29 +58,10 @@ class AllocatorImpl : public Allocator { friend class TextReadoutImpl; friend class NotifyingAllocatorImpl; - struct HeapStatHash { - using is_transparent = void; // NOLINT(readability-identifier-naming) - size_t operator()(const Metric* a) const { return a->statName().hash(); } - size_t operator()(StatName a) const { return a.hash(); } - }; - - struct HeapStatCompare { - using is_transparent = void; // NOLINT(readability-identifier-naming) - bool operator()(const Metric* a, const Metric* b) const { - return a->statName() == b->statName(); - } - bool operator()(const Metric* a, StatName b) const { return a->statName() == b; } - }; - void removeCounterFromSetLockHeld(Counter* counter) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); void removeGaugeFromSetLockHeld(Gauge* gauge) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); void removeTextReadoutFromSetLockHeld(Counter* counter) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - // An unordered set of HeapStatData pointers which keys off the key() - // field in each object. This necessitates a custom comparator and hasher, which key off of the - // StatNamePtr's own StatNamePtrHash and StatNamePtrCompare operators. - template - using StatSet = absl::flat_hash_set; StatSet counters_ ABSL_GUARDED_BY(mutex_); StatSet gauges_ ABSL_GUARDED_BY(mutex_); StatSet text_readouts_ ABSL_GUARDED_BY(mutex_); diff --git a/source/common/stats/metric_impl.h b/source/common/stats/metric_impl.h index c923395b992d..52b577230fd3 100644 --- a/source/common/stats/metric_impl.h +++ b/source/common/stats/metric_impl.h @@ -32,10 +32,35 @@ class MetricHelper { void iterateTagStatNames(const Metric::TagStatNameIterFn& fn) const; void clear(SymbolTable& symbol_table) { stat_names_.clear(symbol_table); } + // Hasher for metrics. + struct Hash { + using is_transparent = void; // NOLINT(readability-identifier-naming) + size_t operator()(const Metric* a) const { return a->statName().hash(); } + size_t operator()(StatName a) const { return a.hash(); } + }; + + // Comparator for metrics. + struct Compare { + using is_transparent = void; // NOLINT(readability-identifier-naming) + bool operator()(const Metric* a, const Metric* b) const { + return a->statName() == b->statName(); + } + bool operator()(const Metric* a, StatName b) const { return a->statName() == b; } + }; + private: StatNameList stat_names_; }; +// An unordered set of stat pointers. which keys off Metric::statName(). +// This necessitates a custom comparator and hasher, using the StatNamePtr's +// own StatNamePtrHash and StatNamePtrCompare operators. +// +// This is used by AllocatorImpl for counters, gauges, and text-readouts, and +// is also used by thread_local_store.h for histograms. +template +using StatSet = absl::flat_hash_set; + /** * Partial implementation of the Metric interface on behalf of Counters, Gauges, * and Histograms. It leaves symbolTable() unimplemented so that implementations diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 697760ed1a4b..54d0c78eba9b 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -63,12 +63,22 @@ void ThreadLocalStoreImpl::setStatsMatcher(StatsMatcherPtr&& stats_matcher) { // in the default_scope. There should be no requests, so there will // be no copies in TLS caches. Thread::LockGuard lock(lock_); + const uint32_t first_histogram_index = deleted_histograms_.size(); for (ScopeImpl* scope : scopes_) { removeRejectedStats(scope->central_cache_->counters_, deleted_counters_); removeRejectedStats(scope->central_cache_->gauges_, deleted_gauges_); removeRejectedStats(scope->central_cache_->histograms_, deleted_histograms_); removeRejectedStats(scope->central_cache_->text_readouts_, deleted_text_readouts_); } + + // Remove any newly rejected histograms from histogram_set_. + { + Thread::LockGuard hist_lock(hist_mutex_); + for (uint32_t i = first_histogram_index; i < deleted_histograms_.size(); ++i) { + uint32_t erased = histogram_set_.erase(deleted_histograms_[i].get()); + ASSERT(erased == 1); + } + } } template @@ -160,16 +170,11 @@ std::vector ThreadLocalStoreImpl::textReadouts() const { std::vector ThreadLocalStoreImpl::histograms() const { std::vector ret; - Thread::LockGuard lock(lock_); - // TODO(ramaraochavali): As histograms don't share storage, there is a chance of duplicate names - // here. We need to create global storage for histograms similar to how we have a central storage - // in shared memory for counters/gauges. In the interim, no de-dup is done here. This may result - // in histograms with duplicate names, but until shared storage is implemented it's ultimately - // less confusing for users who have such configs. - for (ScopeImpl* scope : scopes_) { - for (const auto& name_histogram_pair : scope->central_cache_->histograms_) { - const ParentHistogramSharedPtr& parent_hist = name_histogram_pair.second; - ret.push_back(parent_hist); + Thread::LockGuard lock(hist_mutex_); + { + ret.reserve(histogram_set_.size()); + for (const auto& histogram_ptr : histogram_set_) { + ret.emplace_back(histogram_ptr); } } @@ -189,6 +194,11 @@ void ThreadLocalStoreImpl::initializeThreading(Event::Dispatcher& main_thread_di void ThreadLocalStoreImpl::shutdownThreading() { // This will block both future cache fills as well as cache flushes. shutting_down_ = true; + Thread::LockGuard lock(hist_mutex_); + for (ParentHistogramImpl* histogram : histogram_set_) { + histogram->setShuttingDown(true); + } + histogram_set_.clear(); } void ThreadLocalStoreImpl::mergeHistograms(PostMergeCb merge_complete_cb) { @@ -197,12 +207,9 @@ void ThreadLocalStoreImpl::mergeHistograms(PostMergeCb merge_complete_cb) { merge_in_progress_ = true; tls_->runOnAllThreads( [this]() -> void { - for (const auto& scope : tls_->getTyped().scope_cache_) { - const TlsCacheEntry& tls_cache_entry = scope.second; - for (const auto& name_histogram_pair : tls_cache_entry.histograms_) { - const TlsHistogramSharedPtr& tls_hist = name_histogram_pair.second; - tls_hist->beginMerge(); - } + for (const auto& id_hist : tls_->getTyped().tls_histogram_cache_) { + const TlsHistogramSharedPtr& tls_hist = id_hist.second; + tls_hist->beginMerge(); } }, [this, merge_complete_cb]() -> void { mergeInternal(merge_complete_cb); }); @@ -257,6 +264,10 @@ void ThreadLocalStoreImpl::releaseScopeCrossThread(ScopeImpl* scope) { if (!shutting_down_ && main_thread_dispatcher_) { const uint64_t scope_id = scope->scope_id_; lock.release(); + + // TODO(jmarantz): consider batching all the scope IDs that should be + // cleared from TLS caches to reduce bursts of runOnAllThreads on a large + // config update. See the pattern below used for histograms. main_thread_dispatcher_->post([this, central_cache, scope_id]() { sync_.syncPoint(MainDispatcherCleanupSync); clearScopeFromCaches(scope_id, central_cache); @@ -264,12 +275,27 @@ void ThreadLocalStoreImpl::releaseScopeCrossThread(ScopeImpl* scope) { } } +void ThreadLocalStoreImpl::releaseHistogramCrossThread(uint64_t histogram_id) { + // This can happen from any thread. We post() back to the main thread which will initiate the + // cache flush operation. + if (!shutting_down_ && main_thread_dispatcher_) { + main_thread_dispatcher_->post( + [this, histogram_id]() { clearHistogramFromCaches(histogram_id); }); + } +} + ThreadLocalStoreImpl::TlsCacheEntry& ThreadLocalStoreImpl::TlsCache::insertScope(uint64_t scope_id) { return scope_cache_[scope_id]; } void ThreadLocalStoreImpl::TlsCache::eraseScope(uint64_t scope_id) { scope_cache_.erase(scope_id); } +void ThreadLocalStoreImpl::TlsCache::eraseHistogram(uint64_t histogram_id) { + // This is called for every histogram in every thread, even though the + // histogram may not have been cached in each thread yet. So we don't + // want to check whether the erase() call erased anything. + tls_histogram_cache_.erase(histogram_id); +} void ThreadLocalStoreImpl::clearScopeFromCaches(uint64_t scope_id, CentralCacheEntrySharedPtr central_cache) { @@ -283,6 +309,22 @@ void ThreadLocalStoreImpl::clearScopeFromCaches(uint64_t scope_id, } } +void ThreadLocalStoreImpl::clearHistogramFromCaches(uint64_t histogram_id) { + // If we are shutting down we no longer perform cache flushes as workers may be shutting down + // at the same time. + if (!shutting_down_) { + // Perform a cache flush on all threads. + // + // TODO(jmarantz): If this cross-thread posting proves to be a performance + // bottleneck, + // https://gist.github.com/jmarantz/838cb6de7e74c0970ea6b63eded0139a + // contains a patch that will implement batching together to clear multiple + // histograms. + tls_->runOnAllThreads( + [this, histogram_id]() { tls_->getTyped().eraseHistogram(histogram_id); }); + } +} + ThreadLocalStoreImpl::ScopeImpl::ScopeImpl(ThreadLocalStoreImpl& parent, const std::string& prefix) : scope_id_(parent.next_scope_id_++), parent_(parent), prefix_(Utility::sanitizeStatsName(prefix), parent.alloc_.symbolTable()), @@ -566,9 +608,23 @@ Histogram& ThreadLocalStoreImpl::ScopeImpl::histogramFromStatNameWithTags( [&buckets, this](absl::string_view stat_name) { buckets = &parent_.histogram_settings_->buckets(stat_name); }); - RefcountPtr stat(new ParentHistogramImpl( - final_stat_name, unit, parent_, *this, tag_helper.tagExtractedName(), - tag_helper.statNameTags(), *buckets)); + + RefcountPtr stat; + { + Thread::LockGuard lock(parent_.hist_mutex_); + auto iter = parent_.histogram_set_.find(final_stat_name); + if (iter != parent_.histogram_set_.end()) { + stat = RefcountPtr(*iter); + } else { + stat = new ParentHistogramImpl(final_stat_name, unit, parent_, + tag_helper.tagExtractedName(), tag_helper.statNameTags(), + *buckets, parent_.next_histogram_id_++); + if (!parent_.shutting_down_) { + parent_.histogram_set_.insert(stat.get()); + } + } + } + central_ref = ¢ral_cache_->histograms_[stat->statName()]; *central_ref = stat; } @@ -639,34 +695,34 @@ TextReadoutOptConstRef ThreadLocalStoreImpl::ScopeImpl::findTextReadout(StatName return findStatLockHeld(name, central_cache_->text_readouts_); } -Histogram& ThreadLocalStoreImpl::ScopeImpl::tlsHistogram(StatName name, - ParentHistogramImpl& parent) { +Histogram& ThreadLocalStoreImpl::tlsHistogram(ParentHistogramImpl& parent, uint64_t id) { // tlsHistogram() is generally not called for a histogram that is rejected by // the matcher, so no further rejection-checking is needed at this level. // TlsHistogram inherits its reject/accept status from ParentHistogram. // See comments in counterFromStatName() which explains the logic here. - StatNameHashMap* tls_cache = nullptr; - if (!parent_.shutting_down_ && parent_.tls_) { - tls_cache = &parent_.tls_->getTyped().scope_cache_[this->scope_id_].histograms_; - auto iter = tls_cache->find(name); - if (iter != tls_cache->end()) { - return *iter->second; + TlsHistogramSharedPtr* tls_histogram = nullptr; + if (!shutting_down_ && tls_ != nullptr) { + TlsCache& tls_cache = tls_->getTyped(); + tls_histogram = &tls_cache.tls_histogram_cache_[id]; + if (*tls_histogram != nullptr) { + return **tls_histogram; } } - StatNameTagHelper tag_helper(parent_, name, absl::nullopt); + StatNameTagHelper tag_helper(*this, parent.statName(), absl::nullopt); TlsHistogramSharedPtr hist_tls_ptr( - new ThreadLocalHistogramImpl(name, parent.unit(), tag_helper.tagExtractedName(), + new ThreadLocalHistogramImpl(parent.statName(), parent.unit(), tag_helper.tagExtractedName(), tag_helper.statNameTags(), symbolTable())); parent.addTlsHistogram(hist_tls_ptr); - if (tls_cache) { - tls_cache->insert(std::make_pair(hist_tls_ptr->statName(), hist_tls_ptr)); + if (tls_histogram != nullptr) { + *tls_histogram = hist_tls_ptr; } + return *hist_tls_ptr; } @@ -682,7 +738,7 @@ ThreadLocalHistogramImpl::ThreadLocalHistogramImpl(StatName name, Histogram::Uni } ThreadLocalHistogramImpl::~ThreadLocalHistogramImpl() { - MetricImpl::clear(symbolTable()); + MetricImpl::clear(symbol_table_); hist_free(histograms_[0]); hist_free(histograms_[1]); } @@ -699,28 +755,78 @@ void ThreadLocalHistogramImpl::merge(histogram_t* target) { hist_clear(*other_histogram); } -ParentHistogramImpl::ParentHistogramImpl(StatName name, Histogram::Unit unit, Store& parent, - TlsScope& tls_scope, StatName tag_extracted_name, +ParentHistogramImpl::ParentHistogramImpl(StatName name, Histogram::Unit unit, + ThreadLocalStoreImpl& thread_local_store, + StatName tag_extracted_name, const StatNameTagVector& stat_name_tags, - ConstSupportedBuckets& supported_buckets) - : MetricImpl(name, tag_extracted_name, stat_name_tags, parent.symbolTable()), unit_(unit), - parent_(parent), tls_scope_(tls_scope), interval_histogram_(hist_alloc()), + ConstSupportedBuckets& supported_buckets, uint64_t id) + : MetricImpl(name, tag_extracted_name, stat_name_tags, thread_local_store.symbolTable()), + unit_(unit), thread_local_store_(thread_local_store), interval_histogram_(hist_alloc()), cumulative_histogram_(hist_alloc()), interval_statistics_(interval_histogram_, supported_buckets), - cumulative_statistics_(cumulative_histogram_, supported_buckets), merged_(false) {} + cumulative_statistics_(cumulative_histogram_, supported_buckets), merged_(false), id_(id) {} ParentHistogramImpl::~ParentHistogramImpl() { - MetricImpl::clear(parent_.symbolTable()); + thread_local_store_.releaseHistogramCrossThread(id_); + ASSERT(ref_count_ == 0); + MetricImpl::clear(thread_local_store_.symbolTable()); hist_free(interval_histogram_); hist_free(cumulative_histogram_); } +void ParentHistogramImpl::incRefCount() { ++ref_count_; } + +bool ParentHistogramImpl::decRefCount() { + bool ret; + if (shutting_down_) { + // When shutting down, we cannot reference thread_local_store_, as + // histograms can outlive the store. So we decrement the ref-count without + // the stores' lock. We will not be removing the object from the store's + // histogram map in this scenario, as the set was cleared during shutdown, + // and will not be repopulated in histogramFromStatNameWithTags after + // initiating shutdown. + ret = --ref_count_ == 0; + } else { + // We delegate to the Store object to decrement the ref-count so it can hold + // the lock to the map. If we don't hold a lock, another thread may + // simultaneously try to allocate the same name'd histogram after we + // decrement it, and we'll wind up with a dtor/update race. To avoid this we + // must hold the lock until the histogram is removed from the map. + // + // See also StatsSharedImpl::decRefCount() in allocator_impl.cc, which has + // the same issue. + ret = thread_local_store_.decHistogramRefCount(*this, ref_count_); + } + return ret; +} + +bool ThreadLocalStoreImpl::decHistogramRefCount(ParentHistogramImpl& hist, + std::atomic& ref_count) { + // We must hold the store's histogram lock when decrementing the + // refcount. Otherwise another thread may simultaneously try to allocate the + // same name'd stat after we decrement it, and we'll wind up with a + // dtor/update race. To avoid this we must hold the lock until the stat is + // removed from the map. + Thread::LockGuard lock(hist_mutex_); + ASSERT(ref_count >= 1); + if (--ref_count == 0) { + if (!shutting_down_) { + const size_t count = histogram_set_.erase(hist.statName()); + ASSERT(shutting_down_ || count == 1); + } + return true; + } + return false; +} + +SymbolTable& ParentHistogramImpl::symbolTable() { return thread_local_store_.symbolTable(); } + Histogram::Unit ParentHistogramImpl::unit() const { return unit_; } void ParentHistogramImpl::recordValue(uint64_t value) { - Histogram& tls_histogram = tls_scope_.tlsHistogram(statName(), *this); + Histogram& tls_histogram = thread_local_store_.tlsHistogram(*this, id_); tls_histogram.recordValue(value); - parent_.deliverHistogramToSinks(*this, value); + thread_local_store_.deliverHistogramToSinks(*this, value); } bool ParentHistogramImpl::used() const { diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index c86844a2d38c..22d72bfaa9e0 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -74,16 +74,16 @@ class ThreadLocalHistogramImpl : public HistogramImplHelper { using TlsHistogramSharedPtr = RefcountPtr; -class TlsScope; +class ThreadLocalStoreImpl; /** * Log Linear Histogram implementation that is stored in the main thread. */ class ParentHistogramImpl : public MetricImpl { public: - ParentHistogramImpl(StatName name, Histogram::Unit unit, Store& parent, TlsScope& tls_scope, + ParentHistogramImpl(StatName name, Histogram::Unit unit, ThreadLocalStoreImpl& parent, StatName tag_extracted_name, const StatNameTagVector& stat_name_tags, - ConstSupportedBuckets& supported_buckets); + ConstSupportedBuckets& supported_buckets, uint64_t id); ~ParentHistogramImpl() override; void addTlsHistogram(const TlsHistogramSharedPtr& hist_ptr); @@ -108,20 +108,23 @@ class ParentHistogramImpl : public MetricImpl { const std::string bucketSummary() const override; // Stats::Metric - SymbolTable& symbolTable() override { return parent_.symbolTable(); } + SymbolTable& symbolTable() override; bool used() const override; // RefcountInterface - void incRefCount() override { refcount_helper_.incRefCount(); } - bool decRefCount() override { return refcount_helper_.decRefCount(); } - uint32_t use_count() const override { return refcount_helper_.use_count(); } + void incRefCount() override; + bool decRefCount() override; + uint32_t use_count() const override { return ref_count_; } + + // Indicates that the ThreadLocalStore is shutting down, so no need to clear its histogram_set_. + void setShuttingDown(bool shutting_down) { shutting_down_ = shutting_down; } + bool shuttingDown() const { return shutting_down_; } private: bool usedLockHeld() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(merge_lock_); Histogram::Unit unit_; - Store& parent_; - TlsScope& tls_scope_; + ThreadLocalStoreImpl& thread_local_store_; histogram_t* interval_histogram_; histogram_t* cumulative_histogram_; HistogramStatisticsImpl interval_statistics_; @@ -129,27 +132,13 @@ class ParentHistogramImpl : public MetricImpl { mutable Thread::MutexBasicLockable merge_lock_; std::list tls_histograms_ ABSL_GUARDED_BY(merge_lock_); bool merged_; - RefcountHelper refcount_helper_; + std::atomic shutting_down_{false}; + std::atomic ref_count_{0}; + const uint64_t id_; // Index into TlsCache::histogram_cache_. }; using ParentHistogramImplSharedPtr = RefcountPtr; -/** - * Class used to create ThreadLocalHistogram in the scope. - */ -class TlsScope : public Scope { -public: - ~TlsScope() override = default; - - // TODO(ramaraochavali): Allow direct TLS access for the advanced consumers. - /** - * @return a ThreadLocalHistogram within the scope's namespace. - * @param name name of the histogram with scope prefix attached. - * @param parent the parent histogram. - */ - virtual Histogram& tlsHistogram(StatName name, ParentHistogramImpl& parent) PURE; -}; - /** * Store implementation with thread local caching. For design details see * https://github.com/envoyproxy/envoy/blob/master/source/docs/stats.md @@ -266,6 +255,8 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo void shutdownThreading() override; void mergeHistograms(PostMergeCb merge_cb) override; + Histogram& tlsHistogram(ParentHistogramImpl& parent, uint64_t id); + /** * @return a thread synchronizer object used for controlling thread behavior in tests. */ @@ -276,7 +267,12 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo */ const StatNameSet& wellKnownTags() const { return *well_known_tags_; } + bool decHistogramRefCount(ParentHistogramImpl& histogram, std::atomic& ref_count); + void releaseHistogramCrossThread(uint64_t histogram_id); + private: + friend class ThreadLocalStoreTestingPeer; + template using StatRefMap = StatNameHashMap>; struct TlsCacheEntry { @@ -288,9 +284,18 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo StatRefMap gauges_; StatRefMap text_readouts_; - // The histogram objects are not shared with the central cache, and don't - // require taking a lock when decrementing their ref-count. - StatNameHashMap histograms_; + // Histograms also require holding a mutex while decrementing reference + // counts. The only difference from other stats is that the histogram_set_ + // lives in the ThreadLocalStore object, rather than in + // AllocatorImpl. Histograms are removed from that set when all scopes + // referencing the histogram are dropped. Each ParentHistogram has a unique + // index, which is not re-used during the process lifetime. + // + // There is also a tls_histogram_cache_ in the TlsCache object, which is + // not tied to a scope. It maps from parent histogram's unique index to + // a TlsHistogram. This enables continuity between same-named histograms + // in same-named scopes. That scenario is common when re-creating scopes in + // response to xDS. StatNameHashMap parent_histograms_; // We keep a TLS cache of rejected stat names. This costs memory, but @@ -315,7 +320,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo }; using CentralCacheEntrySharedPtr = RefcountPtr; - struct ScopeImpl : public TlsScope { + struct ScopeImpl : public Scope { ScopeImpl(ThreadLocalStoreImpl& parent, const std::string& prefix); ~ScopeImpl() override; @@ -328,7 +333,6 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo Histogram& histogramFromStatNameWithTags(const StatName& name, StatNameTagVectorOptConstRef tags, Histogram::Unit unit) override; - Histogram& tlsHistogram(StatName name, ParentHistogramImpl& parent) override; TextReadout& textReadoutFromStatNameWithTags(const StatName& name, StatNameTagVectorOptConstRef tags) override; ScopePtr createScope(const std::string& name) override { @@ -437,6 +441,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo struct TlsCache : public ThreadLocal::ThreadLocalObject { TlsCacheEntry& insertScope(uint64_t scope_id); void eraseScope(uint64_t scope_id); + void eraseHistogram(uint64_t histogram); // The TLS scope cache is keyed by scope ID. This is used to avoid complex circular references // during scope destruction. An ID is required vs. using the address of the scope pointer @@ -446,6 +451,9 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo // store. See the overview for more information. This complexity is required for lockless // operation in the fast path. absl::flat_hash_map scope_cache_; + + // Maps from histogram ID (monotonically increasing) to a TLS histogram. + absl::flat_hash_map tls_histogram_cache_; }; template bool iterHelper(StatFn fn) const { @@ -460,6 +468,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo std::string getTagsForName(const std::string& name, TagVector& tags) const; void clearScopeFromCaches(uint64_t scope_id, CentralCacheEntrySharedPtr central_cache); + void clearHistogramFromCaches(uint64_t histogram_id); void releaseScopeCrossThread(ScopeImpl* scope); void mergeInternal(PostMergeCb merge_cb); bool rejects(StatName name) const; @@ -497,15 +506,19 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo // It seems like it would be better to have each client that expects a stat // to exist to hold it as (e.g.) a CounterSharedPtr rather than a Counter& // but that would be fairly complex to change. - std::vector deleted_counters_; - std::vector deleted_gauges_; - std::vector deleted_histograms_; - std::vector deleted_text_readouts_; + std::vector deleted_counters_ ABSL_GUARDED_BY(lock_); + std::vector deleted_gauges_ ABSL_GUARDED_BY(lock_); + std::vector deleted_histograms_ ABSL_GUARDED_BY(lock_); + std::vector deleted_text_readouts_ ABSL_GUARDED_BY(lock_); Thread::ThreadSynchronizer sync_; std::atomic next_scope_id_{}; + uint64_t next_histogram_id_ ABSL_GUARDED_BY(hist_mutex_) = 0; StatNameSetPtr well_known_tags_; + + mutable Thread::MutexBasicLockable hist_mutex_; + StatSet histogram_set_ ABSL_GUARDED_BY(hist_mutex_); }; using ThreadLocalStoreImplPtr = std::unique_ptr; diff --git a/source/docs/stats.md b/source/docs/stats.md index 43be6992146c..f80d1b46932f 100644 --- a/source/docs/stats.md +++ b/source/docs/stats.md @@ -16,7 +16,7 @@ values, they are passed from parent to child in an RPC protocol. They were previously held in shared memory, which imposed various restrictions. Unlike the shared memory implementation, the RPC passing *requires a mode-bit specified when constructing gauges indicating whether it should be accumulated across hot-restarts*. - + ## Performance and Thread Local Storage A key tenant of the Envoy architecture is high performance on machines with @@ -77,6 +77,18 @@ followed. accumulates in to *interval* histograms. * Finally the main *interval* histogram is merged to *cumulative* histogram. +`ParentHistogram`s are held weakly a set in ThreadLocalStore. Like other stats, +they keep an embedded reference count and are removed from the set and destroyed +when the last strong reference disappears. Consequently, we must hold a lock for +the set when decrementing histogram reference counts. A similar process occurs for +other types of stats, but in those cases it is taken care of in `AllocatorImpl`. +There are strong references to `ParentHistograms` in TlsCacheEntry::parent_histograms_. + +Thread-local `TlsHistogram`s are created on behalf of a `ParentHistogram` +whenever accessed from a worker thread. They are strongly referenced in the +`ParentHistogram` as well as in a cache in the `ThreadLocalStore`, to help +maintain data continuity as scopes are re-created during operation. + ## Stat naming infrastructure and memory consumption Stat names are replicated in several places in various forms. diff --git a/source/server/admin/stats_handler.cc b/source/server/admin/stats_handler.cc index 753774f59dc9..e64fd878a8fb 100644 --- a/source/server/admin/stats_handler.cc +++ b/source/server/admin/stats_handler.cc @@ -121,13 +121,11 @@ Http::Code StatsHandler::handlerStats(absl::string_view url, for (const auto& stat : all_stats) { response.add(fmt::format("{}: {}\n", stat.first, stat.second)); } - // TODO(ramaraochavali): See the comment in ThreadLocalStoreImpl::histograms() for why we use a - // multimap here. This makes sure that duplicate histograms get output. When shared storage is - // implemented this can be switched back to a normal map. - std::multimap all_histograms; + std::map all_histograms; for (const Stats::ParentHistogramSharedPtr& histogram : server_.stats().histograms()) { if (shouldShowMetric(*histogram, used_only, regex)) { - all_histograms.emplace(histogram->name(), histogram->quantileSummary()); + auto insert = all_histograms.emplace(histogram->name(), histogram->quantileSummary()); + ASSERT(insert.second); // No duplicates expected. } } for (const auto& histogram : all_histograms) { diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 726f32174ae8..135c6b424097 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -29,6 +29,7 @@ #include "gtest/gtest.h" using testing::_; +using testing::HasSubstr; using testing::InSequence; using testing::NiceMock; using testing::Ref; @@ -39,6 +40,30 @@ namespace Stats { const uint64_t MaxStatNameLength = 127; +class ThreadLocalStoreTestingPeer { +public: + // Calculates the number of TLS histograms across all threads. This requires + // dispatching to all threads and blocking on their completion, and is exposed + // as a testing peer to enable tests that ensure that TLS histograms don't + // leak. + // + // Note that this must be called from the "main thread", which has different + // implications for unit tests that use real threads vs mocks. The easiest way + // to capture this in a general purpose helper is to use a callback to convey + // the resultant sum. + static void numTlsHistograms(ThreadLocalStoreImpl& thread_local_store_impl, + const std::function& num_tls_hist_cb) { + auto num_tls_histograms = std::make_shared>(0); + thread_local_store_impl.tls_->runOnAllThreads( + [&thread_local_store_impl, num_tls_histograms]() { + auto& tls_cache = + thread_local_store_impl.tls_->getTyped(); + *num_tls_histograms += tls_cache.tls_histogram_cache_.size(); + }, + [num_tls_hist_cb, num_tls_histograms]() { num_tls_hist_cb(*num_tls_histograms); }); + } +}; + class StatsThreadLocalStoreTest : public testing::Test { public: StatsThreadLocalStoreTest() @@ -52,6 +77,21 @@ class StatsThreadLocalStoreTest : public testing::Test { store_->addSink(sink_); } + uint32_t numTlsHistograms() { + uint32_t num_tls_histograms; + absl::Mutex mutex; + bool done = false; + ThreadLocalStoreTestingPeer::numTlsHistograms( + *store_, [&mutex, &done, &num_tls_histograms](uint32_t num) { + absl::MutexLock lock(&mutex); + num_tls_histograms = num; + done = true; + }); + absl::MutexLock lock(&mutex); + mutex.Await(absl::Condition(&done)); + return num_tls_histograms; + } + SymbolTablePtr symbol_table_; NiceMock main_thread_dispatcher_; NiceMock tls_; @@ -381,6 +421,52 @@ TEST_F(StatsThreadLocalStoreTest, BasicScope) { store_->shutdownThreading(); scope1->deliverHistogramToSinks(h1, 100); scope1->deliverHistogramToSinks(h2, 200); + scope1.reset(); + tls_.shutdownThread(); +} + +TEST_F(StatsThreadLocalStoreTest, HistogramScopeOverlap) { + InSequence s; + store_->initializeThreading(main_thread_dispatcher_, tls_); + + // Creating two scopes with the same name gets you two distinct scope objects. + ScopePtr scope1 = store_->createScope("scope."); + ScopePtr scope2 = store_->createScope("scope."); + EXPECT_NE(scope1, scope2); + + EXPECT_EQ(0, store_->histograms().size()); + EXPECT_EQ(0, numTlsHistograms()); + + // However, stats created in the two same-named scopes will be the same objects. + Counter& counter = scope1->counterFromString("counter"); + EXPECT_EQ(&counter, &scope2->counterFromString("counter")); + Gauge& gauge = scope1->gaugeFromString("gauge", Gauge::ImportMode::Accumulate); + EXPECT_EQ(&gauge, &scope2->gaugeFromString("gauge", Gauge::ImportMode::Accumulate)); + TextReadout& text_readout = scope1->textReadoutFromString("tr"); + EXPECT_EQ(&text_readout, &scope2->textReadoutFromString("tr")); + Histogram& histogram = scope1->histogramFromString("histogram", Histogram::Unit::Unspecified); + EXPECT_EQ(&histogram, &scope2->histogramFromString("histogram", Histogram::Unit::Unspecified)); + + // The histogram was created in scope1, which can now be destroyed. But the + // histogram is kept alive by scope2. + EXPECT_CALL(sink_, onHistogramComplete(Ref(histogram), 100)); + histogram.recordValue(100); + EXPECT_EQ(1, store_->histograms().size()); + EXPECT_EQ(1, numTlsHistograms()); + scope1.reset(); + EXPECT_EQ(1, store_->histograms().size()); + EXPECT_EQ(1, numTlsHistograms()); + EXPECT_CALL(sink_, onHistogramComplete(Ref(histogram), 200)); + histogram.recordValue(200); + EXPECT_EQ(&histogram, &scope2->histogramFromString("histogram", Histogram::Unit::Unspecified)); + scope2.reset(); + EXPECT_EQ(0, store_->histograms().size()); + EXPECT_EQ(0, numTlsHistograms()); + + store_->shutdownThreading(); + + store_->histogramFromString("histogram_after_shutdown", Histogram::Unit::Unspecified); + tls_.shutdownThread(); } @@ -1102,7 +1188,7 @@ TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsFakeSymbolTable) { TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( 100, [this](absl::string_view name) { store_->counterFromString(std::string(name)); }); - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 1498160); // Apr 8, 2020 + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 1498128); // July 30, 2020 EXPECT_MEMORY_LE(memory_test.consumedBytes(), 1.6 * million_); } @@ -1122,7 +1208,7 @@ TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsRealSymbolTable) { TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( 100, [this](absl::string_view name) { store_->counterFromString(std::string(name)); }); - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 827664); // July 2, 2020 + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 827632); // July 20, 2020 EXPECT_MEMORY_LE(memory_test.consumedBytes(), 0.9 * million_); } @@ -1378,9 +1464,8 @@ TEST_F(HistogramTest, ParentHistogramBucketSummary) { parent_histogram->bucketSummary()); } -class ClusterShutdownCleanupStarvationTest : public ThreadLocalStoreNoMocksTestBase { -public: - static constexpr uint32_t NumThreads = 2; +class ThreadLocalRealThreadsTestBase : public ThreadLocalStoreNoMocksTestBase { +protected: static constexpr uint32_t NumScopes = 1000; static constexpr uint32_t NumIters = 35; @@ -1416,18 +1501,17 @@ class ClusterShutdownCleanupStarvationTest : public ThreadLocalStoreNoMocksTestB absl::BlockingCounter blocking_counter_; }; - ClusterShutdownCleanupStarvationTest() - : start_time_(time_system_.monotonicTime()), api_(Api::createApiForTest()), - thread_factory_(api_->threadFactory()), pool_(store_->symbolTable()), - my_counter_name_(pool_.add("my_counter")), - my_counter_scoped_name_(pool_.add("scope.my_counter")) { + ThreadLocalRealThreadsTestBase(uint32_t num_threads) + : num_threads_(num_threads), start_time_(time_system_.monotonicTime()), + api_(Api::createApiForTest()), thread_factory_(api_->threadFactory()), + pool_(store_->symbolTable()) { // This is the same order as InstanceImpl::initialize in source/server/server.cc. - thread_dispatchers_.resize(NumThreads); + thread_dispatchers_.resize(num_threads_); { - BlockingBarrier blocking_barrier(NumThreads + 1); + BlockingBarrier blocking_barrier(num_threads_ + 1); main_thread_ = thread_factory_.createThread( [this, &blocking_barrier]() { mainThreadFn(blocking_barrier); }); - for (uint32_t i = 0; i < NumThreads; ++i) { + for (uint32_t i = 0; i < num_threads_; ++i) { threads_.emplace_back(thread_factory_.createThread( [this, i, &blocking_barrier]() { workerThreadFn(i, blocking_barrier); })); } @@ -1447,7 +1531,7 @@ class ClusterShutdownCleanupStarvationTest : public ThreadLocalStoreNoMocksTestB } } - ~ClusterShutdownCleanupStarvationTest() override { + ~ThreadLocalRealThreadsTestBase() override { { BlockingBarrier blocking_barrier(1); main_dispatcher_->post(blocking_barrier.run([this]() { @@ -1473,14 +1557,6 @@ class ClusterShutdownCleanupStarvationTest : public ThreadLocalStoreNoMocksTestB main_thread_->join(); } - void createScopesIncCountersAndCleanup() { - for (uint32_t i = 0; i < NumScopes; ++i) { - ScopePtr scope = store_->createScope("scope."); - Counter& counter = scope->counterFromStatName(my_counter_name_); - counter.inc(); - } - } - void workerThreadFn(uint32_t thread_index, BlockingBarrier& blocking_barrier) { thread_dispatchers_[thread_index] = api_->allocateDispatcher(absl::StrCat("test_worker_", thread_index)); @@ -1494,19 +1570,21 @@ class ClusterShutdownCleanupStarvationTest : public ThreadLocalStoreNoMocksTestB main_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); } - void createScopesIncCountersAndCleanupAllThreads() { - BlockingBarrier blocking_barrier(NumThreads); - for (Event::DispatcherPtr& thread_dispatcher : thread_dispatchers_) { - thread_dispatcher->post( - blocking_barrier.run([this]() { createScopesIncCountersAndCleanup(); })); - } + void mainDispatchBlock() { + // To ensure all stats are freed we have to wait for a few posts() to clear. + // First, wait for the main-dispatcher to initiate the cross-thread TLS cleanup. + BlockingBarrier blocking_barrier(1); + main_dispatcher_->post(blocking_barrier.run([]() {})); } - std::chrono::seconds elapsedTime() { - return std::chrono::duration_cast(time_system_.monotonicTime() - - start_time_); + void tlsBlock() { + BlockingBarrier blocking_barrier(num_threads_); + for (Event::DispatcherPtr& thread_dispatcher : thread_dispatchers_) { + thread_dispatcher->post(blocking_barrier.run([]() {})); + } } + const uint32_t num_threads_; Event::TestRealTimeSystem time_system_; MonotonicTime start_time_; Api::ApiPtr api_; @@ -1517,6 +1595,37 @@ class ClusterShutdownCleanupStarvationTest : public ThreadLocalStoreNoMocksTestB Thread::ThreadPtr main_thread_; std::vector threads_; StatNamePool pool_; +}; + +class ClusterShutdownCleanupStarvationTest : public ThreadLocalRealThreadsTestBase { +protected: + static constexpr uint32_t NumThreads = 2; + + ClusterShutdownCleanupStarvationTest() + : ThreadLocalRealThreadsTestBase(NumThreads), my_counter_name_(pool_.add("my_counter")), + my_counter_scoped_name_(pool_.add("scope.my_counter")) {} + + void createScopesIncCountersAndCleanup() { + for (uint32_t i = 0; i < NumScopes; ++i) { + ScopePtr scope = store_->createScope("scope."); + Counter& counter = scope->counterFromStatName(my_counter_name_); + counter.inc(); + } + } + + void createScopesIncCountersAndCleanupAllThreads() { + BlockingBarrier blocking_barrier(NumThreads); + for (Event::DispatcherPtr& thread_dispatcher : thread_dispatchers_) { + thread_dispatcher->post( + blocking_barrier.run([this]() { createScopesIncCountersAndCleanup(); })); + } + } + + std::chrono::seconds elapsedTime() { + return std::chrono::duration_cast(time_system_.monotonicTime() - + start_time_); + } + StatName my_counter_name_; StatName my_counter_scoped_name_; }; @@ -1529,24 +1638,14 @@ TEST_F(ClusterShutdownCleanupStarvationTest, TwelveThreadsWithBlockade) { for (uint32_t i = 0; i < NumIters && elapsedTime() < std::chrono::seconds(5); ++i) { createScopesIncCountersAndCleanupAllThreads(); - // To ensure all stats are freed we have to wait for a few posts() to clear. // First, wait for the main-dispatcher to initiate the cross-thread TLS cleanup. - auto main_dispatch_block = [this]() { - BlockingBarrier blocking_barrier(1); - main_dispatcher_->post(blocking_barrier.run([]() {})); - }; - main_dispatch_block(); + mainDispatchBlock(); // Next, wait for all the worker threads to complete their TLS cleanup. - { - BlockingBarrier blocking_barrier(NumThreads); - for (Event::DispatcherPtr& thread_dispatcher : thread_dispatchers_) { - thread_dispatcher->post(blocking_barrier.run([]() {})); - } - } + tlsBlock(); // Finally, wait for the final central-cache cleanup, which occurs on the main thread. - main_dispatch_block(); + mainDispatchBlock(); // Here we show that the counter cleanups have finished, because the use-count is 1. CounterSharedPtr counter = @@ -1583,5 +1682,124 @@ TEST_F(ClusterShutdownCleanupStarvationTest, TwelveThreadsWithoutBlockade) { store_->sync().signal(ThreadLocalStoreImpl::MainDispatcherCleanupSync); } +class HistogramThreadTest : public ThreadLocalRealThreadsTestBase { +protected: + static constexpr uint32_t NumThreads = 10; + + HistogramThreadTest() : ThreadLocalRealThreadsTestBase(NumThreads) {} + + void mergeHistograms() { + BlockingBarrier blocking_barrier(1); + main_dispatcher_->post([this, &blocking_barrier]() { + store_->mergeHistograms(blocking_barrier.decrementCountFn()); + }); + } + + uint32_t numTlsHistograms() { + uint32_t num; + { + BlockingBarrier blocking_barrier(1); + main_dispatcher_->post([this, &num, &blocking_barrier]() { + ThreadLocalStoreTestingPeer::numTlsHistograms(*store_, + [&num, &blocking_barrier](uint32_t num_hist) { + num = num_hist; + blocking_barrier.decrementCount(); + }); + }); + } + return num; + } + + // Executes a function on every worker thread dispatcher. + void foreachThread(const std::function& fn) { + BlockingBarrier blocking_barrier(NumThreads); + for (Event::DispatcherPtr& thread_dispatcher : thread_dispatchers_) { + thread_dispatcher->post(blocking_barrier.run(fn)); + } + } +}; + +TEST_F(HistogramThreadTest, MakeHistogramsAndRecordValues) { + foreachThread([this]() { + Histogram& histogram = + store_->histogramFromString("my_hist", Stats::Histogram::Unit::Unspecified); + histogram.recordValue(42); + }); + + mergeHistograms(); + + auto histograms = store_->histograms(); + ASSERT_EQ(1, histograms.size()); + ParentHistogramSharedPtr hist = histograms[0]; + EXPECT_THAT(hist->bucketSummary(), + HasSubstr(absl::StrCat(" B25(0,0) B50(", NumThreads, ",", NumThreads, ") "))); +} + +TEST_F(HistogramThreadTest, ScopeOverlap) { + // Creating two scopes with the same name gets you two distinct scope objects. + ScopePtr scope1 = store_->createScope("scope."); + ScopePtr scope2 = store_->createScope("scope."); + EXPECT_NE(scope1, scope2); + + EXPECT_EQ(0, store_->histograms().size()); + EXPECT_EQ(0, numTlsHistograms()); + + // Histograms created in the two same-named scopes will be the same objects. + foreachThread([&scope1, &scope2]() { + Histogram& histogram = scope1->histogramFromString("histogram", Histogram::Unit::Unspecified); + EXPECT_EQ(&histogram, &scope2->histogramFromString("histogram", Histogram::Unit::Unspecified)); + histogram.recordValue(100); + }); + + mergeHistograms(); + + // Verify that we have the expected number of TLS histograms since we accessed + // the histogram on every thread. + std::vector histograms = store_->histograms(); + ASSERT_EQ(1, histograms.size()); + EXPECT_EQ(NumThreads, numTlsHistograms()); + + // There's no convenient API to pull data out of the histogram, except as + // a string. This expectation captures the bucket transition to indicate + // 0 samples at less than 100, and 10 between 100 and 249 inclusive. + EXPECT_THAT(histograms[0]->bucketSummary(), + HasSubstr(absl::StrCat(" B100(0,0) B250(", NumThreads, ",", NumThreads, ") "))); + + // The histogram was created in scope1, which can now be destroyed. But the + // histogram is kept alive by scope2. + scope1.reset(); + histograms = store_->histograms(); + EXPECT_EQ(1, histograms.size()); + EXPECT_EQ(NumThreads, numTlsHistograms()); + + // We can continue to accumulate samples at the scope2's view of the same + // histogram, and they will combine with the existing data, despite the + // fact that scope1 has been deleted. + foreachThread([&scope2]() { + Histogram& histogram = scope2->histogramFromString("histogram", Histogram::Unit::Unspecified); + histogram.recordValue(300); + }); + + mergeHistograms(); + + // Shows the bucket summary with 10 samples at >=100, and 20 at >=250. + EXPECT_THAT(histograms[0]->bucketSummary(), + HasSubstr(absl::StrCat(" B100(0,0) B250(0,", NumThreads, ") B500(", NumThreads, ",", + 2 * NumThreads, ") "))); + + // Now clear everything, and synchronize the system by calling mergeHistograms(). + // THere should be no more ParentHistograms or TlsHistograms. + scope2.reset(); + histograms.clear(); + mergeHistograms(); + + EXPECT_EQ(0, store_->histograms().size()); + EXPECT_EQ(0, numTlsHistograms()); + + store_->shutdownThreading(); + + store_->histogramFromString("histogram_after_shutdown", Histogram::Unit::Unspecified); +} + } // namespace Stats } // namespace Envoy diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 26143f370000..f66a9b8a14bf 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -287,6 +287,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // and body sizes. // 2020/07/21 12034 44811 46000 Add configurable histogram buckets. // 2020/07/31 12035 45002 46000 Init manager store unready targets in hash map. + // 2020/08/10 12275 44949 46000 Re-organize tls histogram maps to improve continuity. // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -304,7 +305,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // We only run the exact test for ipv6 because ipv4 in some cases may allocate a // different number of bytes. We still run the approximate test. if (ip_version_ != Network::Address::IpVersion::v6) { - EXPECT_MEMORY_EQ(m_per_cluster, 45002); + // https://github.com/envoyproxy/envoy/issues/12209 + // EXPECT_MEMORY_EQ(m_per_cluster, 44949); } EXPECT_MEMORY_LE(m_per_cluster, 46000); // Round up to allow platform variations. } @@ -363,6 +365,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // and body sizes. // 2020/07/21 12034 36923 38000 Add configurable histogram buckets. // 2020/07/31 12035 37114 38000 Init manager store unready targets in hash map. + // 2020/08/10 12275 37061 38000 Re-organize tls histogram maps to improve continuity. // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -380,7 +383,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // We only run the exact test for ipv6 because ipv4 in some cases may allocate a // different number of bytes. We still run the approximate test. if (ip_version_ != Network::Address::IpVersion::v6) { - EXPECT_MEMORY_EQ(m_per_cluster, 37114); + // https://github.com/envoyproxy/envoy/issues/12209 + // EXPECT_MEMORY_EQ(m_per_cluster, 37061); } EXPECT_MEMORY_LE(m_per_cluster, 38000); // Round up to allow platform variations. } From 7cce05af396bcd52e140b34a77de8e520bd1f457 Mon Sep 17 00:00:00 2001 From: Elisha Ziskind Date: Mon, 10 Aug 2020 22:03:57 -0400 Subject: [PATCH 25/67] Add missing header files (#12581) Signed-off-by: Elisha Ziskind --- source/common/http/async_client_utility.cc | 2 ++ source/common/network/filter_matcher.cc | 4 +++- source/common/signal/BUILD | 3 +++ source/common/signal/fatal_error_handler.cc | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/source/common/http/async_client_utility.cc b/source/common/http/async_client_utility.cc index 17124f06fb34..664a0fc0c651 100644 --- a/source/common/http/async_client_utility.cc +++ b/source/common/http/async_client_utility.cc @@ -1,5 +1,7 @@ #include "common/http/async_client_utility.h" +#include "common/common/assert.h" + namespace Envoy { namespace Http { diff --git a/source/common/network/filter_matcher.cc b/source/common/network/filter_matcher.cc index 6668850db44e..7b2831b8a55e 100644 --- a/source/common/network/filter_matcher.cc +++ b/source/common/network/filter_matcher.cc @@ -2,6 +2,8 @@ #include "envoy/network/filter.h" +#include "common/common/assert.h" + #include "absl/strings/str_format.h" namespace Envoy { @@ -50,4 +52,4 @@ bool ListenerFilterAndMatcher::matches(ListenerFilterCallbacks& cb) const { } } // namespace Network -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/common/signal/BUILD b/source/common/signal/BUILD index 3008c01cb50e..2a18144c87db 100644 --- a/source/common/signal/BUILD +++ b/source/common/signal/BUILD @@ -12,6 +12,9 @@ envoy_cc_library( name = "fatal_error_handler_lib", srcs = ["fatal_error_handler.cc"], hdrs = ["fatal_error_handler.h"], + deps = [ + "//source/common/common:macros", + ], ) envoy_cc_library( diff --git a/source/common/signal/fatal_error_handler.cc b/source/common/signal/fatal_error_handler.cc index b215d158b158..125093e3c589 100644 --- a/source/common/signal/fatal_error_handler.cc +++ b/source/common/signal/fatal_error_handler.cc @@ -2,6 +2,8 @@ #include +#include "common/common/macros.h" + #include "absl/base/attributes.h" #include "absl/synchronization/mutex.h" From 573170bb77c885278955b2f0d44dbae9875828c6 Mon Sep 17 00:00:00 2001 From: Lisa Lu Date: Mon, 10 Aug 2020 19:07:14 -0700 Subject: [PATCH 26/67] Revert buffer filter visibility back to public (#12579) Commit Message: Revert buffer filter visibility back to public Additional Description: After bringing in #12337, we are unable to build the router check tool as we build it with the buffer filter extension, which is no longer visible to the target we use. This change reverts the visibility change for the buffer filter back to public. Risk Level: Low Testing: N/A Docs Changes: N/A Release Notes: N/A Part of #12444 Signed-off-by: Lisa Lu --- source/extensions/filters/http/buffer/BUILD | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/extensions/filters/http/buffer/BUILD b/source/extensions/filters/http/buffer/BUILD index 9f9364576031..c39db2ac9a85 100644 --- a/source/extensions/filters/http/buffer/BUILD +++ b/source/extensions/filters/http/buffer/BUILD @@ -39,10 +39,6 @@ envoy_cc_extension( hdrs = ["config.h"], security_posture = "robust_to_untrusted_downstream", # Legacy test use. TODO(#9953) clean up. - visibility = [ - "//:extension_config", - "//test:__subpackages__", - ], deps = [ "//include/envoy/registry", "//source/extensions/filters/http:well_known_names", From 689bd37806efb3e8bba3feccc1bc09a8e3270d4a Mon Sep 17 00:00:00 2001 From: Jose Ulises Nino Rivera Date: Mon, 10 Aug 2020 21:10:53 -0700 Subject: [PATCH 27/67] buffer impl: add cast for android compilation (#12583) Commit Message: add cast for android compilation Risk Level: low using the expected type for the constructor as the static_cast type. Testing: local build of envoy mobile for android. CI Signed-off-by: Jose Nino --- source/common/buffer/buffer_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 92ff88742dc1..f5cea7650421 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -40,7 +40,7 @@ class Slice : public SliceData { // SliceData absl::Span getMutableData() override { RELEASE_ASSERT(isMutable(), "Not allowed to call getMutableData if slice is immutable"); - return {base_ + data_, reservable_ - data_}; + return {base_ + data_, static_cast::size_type>(reservable_ - data_)}; } /** From 89b594e09e2f4ebaac43c6b38d778a3e0dc13a3a Mon Sep 17 00:00:00 2001 From: David Raskin <66272127+davidraskin@users.noreply.github.com> Date: Mon, 10 Aug 2020 23:51:19 -0500 Subject: [PATCH 28/67] api: Add log action to RBAC filter api (#11705) The log action will be used to set the dynamic metadata key "envoy.log", which can be used to decide whether to log a request. Signed-off-by: davidraskin --- api/envoy/config/rbac/v3/rbac.proto | 108 +++++++++++------- api/envoy/config/rbac/v4alpha/rbac.proto | 108 +++++++++++------- .../advanced/well_known_dynamic_metadata.rst | 21 ++++ .../http/http_filters/rbac_filter.rst | 3 + .../listeners/network_filters/rbac_filter.rst | 3 + docs/root/version_history/current.rst | 1 + .../envoy/config/rbac/v3/rbac.proto | 108 +++++++++++------- .../envoy/config/rbac/v4alpha/rbac.proto | 108 +++++++++++------- .../extensions/filters/common/rbac/engine.h | 20 ++-- .../filters/common/rbac/engine_impl.cc | 57 ++++++--- .../filters/common/rbac/engine_impl.h | 34 +++++- .../extensions/filters/common/rbac/utility.h | 21 +--- .../filters/http/rbac/rbac_filter.cc | 7 +- .../filters/network/rbac/rbac_filter.cc | 6 +- .../filters/common/rbac/engine_impl_test.cc | 107 ++++++++++++++--- test/extensions/filters/common/rbac/mocks.h | 13 ++- .../http/rbac/rbac_filter_integration_test.cc | 37 ++++++ .../filters/http/rbac/rbac_filter_test.cc | 90 ++++++++++++++- .../filters/network/rbac/filter_test.cc | 71 +++++++++++- 19 files changed, 677 insertions(+), 246 deletions(-) diff --git a/api/envoy/config/rbac/v3/rbac.proto b/api/envoy/config/rbac/v3/rbac.proto index 10520b1ba38f..278e6857603f 100644 --- a/api/envoy/config/rbac/v3/rbac.proto +++ b/api/envoy/config/rbac/v3/rbac.proto @@ -24,8 +24,14 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Role Based Access Control (RBAC)] // Role Based Access Control (RBAC) provides service-level and method-level access control for a -// service. RBAC policies are additive. The policies are examined in order. A request is allowed -// once a matching policy is found (suppose the `action` is ALLOW). +// service. RBAC policies are additive. The policies are examined in order. Requests are allowed +// or denied based on the `action` and whether a matching policy is found. For instance, if the +// action is ALLOW and a matching policy is found the request should be allowed. +// +// RBAC can also be used to make access logging decisions by communicating with access loggers +// through dynamic metadata. When the action is LOG and at least one policy matches, the +// `access_log_hint` value in the shared key namespace 'envoy.common' is set to `true` indicating +// the request should be logged. // // Here is an example of RBAC configuration. It has two policies: // @@ -68,39 +74,55 @@ message RBAC { // Should we do safe-list or block-list style access control? enum Action { - // The policies grant access to principals. The rest is denied. This is safe-list style + // The policies grant access to principals. The rest are denied. This is safe-list style // access control. This is the default type. ALLOW = 0; - // The policies deny access to principals. The rest is allowed. This is block-list style + // The policies deny access to principals. The rest are allowed. This is block-list style // access control. DENY = 1; + + // The policies set the `access_log_hint` dynamic metadata key based on if requests match. + // All requests are allowed. + LOG = 2; } - // The action to take if a policy matches. The request is allowed if and only if: + // The action to take if a policy matches. Every action either allows or denies a request, + // and can also carry out action-specific operations. + // + // Actions: + // + // * ALLOW: Allows the request if and only if there is a policy that matches + // the request. + // * DENY: Allows the request if and only if there are no policies that + // match the request. + // * LOG: Allows all requests. If at least one policy matches, the dynamic + // metadata key `access_log_hint` is set to the value `true` under the shared + // key namespace 'envoy.common'. If no policies match, it is set to `false`. + // Other actions do not modify this key. // - // * `action` is "ALLOWED" and at least one policy matches - // * `action` is "DENY" and none of the policies match Action action = 1; // Maps from policy name to policy. A match occurs when at least one policy matches the request. map policies = 2; } -// Policy specifies a role and the principals that are assigned/denied the role. A policy matches if -// and only if at least one of its permissions match the action taking place AND at least one of its -// principals match the downstream AND the condition is true if specified. +// Policy specifies a role and the principals that are assigned/denied the role. +// A policy matches if and only if at least one of its permissions match the +// action taking place AND at least one of its principals match the downstream +// AND the condition is true if specified. message Policy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v2.Policy"; - // Required. The set of permissions that define a role. Each permission is matched with OR - // semantics. To match all actions for this policy, a single Permission with the `any` field set - // to true should be used. + // Required. The set of permissions that define a role. Each permission is + // matched with OR semantics. To match all actions for this policy, a single + // Permission with the `any` field set to true should be used. repeated Permission permissions = 1 [(validate.rules).repeated = {min_items: 1}]; - // Required. The set of principals that are assigned/denied the role based on “action”. Each - // principal is matched with OR semantics. To match all downstreams for this policy, a single - // Principal with the `any` field set to true should be used. + // Required. The set of principals that are assigned/denied the role based on + // “action”. Each principal is matched with OR semantics. To match all + // downstreams for this policy, a single Principal with the `any` field set to + // true should be used. repeated Principal principals = 2 [(validate.rules).repeated = {min_items: 1}]; // An optional symbolic expression specifying an access control @@ -161,9 +183,9 @@ message Permission { // Metadata that describes additional information about the action. type.matcher.v3.MetadataMatcher metadata = 7; - // Negates matching the provided permission. For instance, if the value of `not_rule` would - // match, this permission would not match. Conversely, if the value of `not_rule` would not - // match, this permission would match. + // Negates matching the provided permission. For instance, if the value of + // `not_rule` would match, this permission would not match. Conversely, if + // the value of `not_rule` would not match, this permission would match. Permission not_rule = 8; // The request server from the client's connection request. This is @@ -176,7 +198,8 @@ message Permission { // // * If the :ref:`TLS Inspector ` // filter is not added, and if a `FilterChainMatch` is not defined for - // the :ref:`server name `, + // the :ref:`server name + // `, // a TLS connection's requested SNI server name will be treated as if it // wasn't present. // @@ -189,13 +212,14 @@ message Permission { } } -// Principal defines an identity or a group of identities for a downstream subject. +// Principal defines an identity or a group of identities for a downstream +// subject. // [#next-free-field: 12] message Principal { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v2.Principal"; - // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. Depending on the context, - // each are applied with the associated behavior. + // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. + // Depending on the context, each are applied with the associated behavior. message Set { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v2.Principal.Set"; @@ -210,19 +234,21 @@ message Principal { reserved 1; - // The name of the principal. If set, The URI SAN or DNS SAN in that order is used from the - // certificate, otherwise the subject field is used. If unset, it applies to any user that is - // authenticated. + // The name of the principal. If set, The URI SAN or DNS SAN in that order + // is used from the certificate, otherwise the subject field is used. If + // unset, it applies to any user that is authenticated. type.matcher.v3.StringMatcher principal_name = 2; } oneof identifier { option (validate.required) = true; - // A set of identifiers that all must match in order to define the downstream. + // A set of identifiers that all must match in order to define the + // downstream. Set and_ids = 1; - // A set of identifiers at least one must match in order to define the downstream. + // A set of identifiers at least one must match in order to define the + // downstream. Set or_ids = 2; // When any is set, it matches any downstream. @@ -237,21 +263,23 @@ message Principal { // A CIDR block that describes the downstream remote/origin address. // Note: This is always the physical peer even if the - // :ref:`remote_ip ` is inferred - // from for example the x-forwarder-for header, proxy protocol, etc. + // :ref:`remote_ip ` is + // inferred from for example the x-forwarder-for header, proxy protocol, + // etc. core.v3.CidrRange direct_remote_ip = 10; // A CIDR block that describes the downstream remote/origin address. // Note: This may not be the physical peer and could be different from the - // :ref:`direct_remote_ip `. - // E.g, if the remote ip is inferred from for example the x-forwarder-for header, - // proxy protocol, etc. + // :ref:`direct_remote_ip + // `. E.g, if the + // remote ip is inferred from for example the x-forwarder-for header, proxy + // protocol, etc. core.v3.CidrRange remote_ip = 11; - // A header (or pseudo-header such as :path or :method) on the incoming HTTP request. Only - // available for HTTP request. - // Note: the pseudo-header :path includes the query and fragment string. Use the `url_path` - // field if you want to match the URL path without the query and fragment string. + // A header (or pseudo-header such as :path or :method) on the incoming HTTP + // request. Only available for HTTP request. Note: the pseudo-header :path + // includes the query and fragment string. Use the `url_path` field if you + // want to match the URL path without the query and fragment string. route.v3.HeaderMatcher header = 6; // A URL path on the incoming HTTP request. Only available for HTTP. @@ -260,9 +288,9 @@ message Principal { // Metadata that describes additional information about the principal. type.matcher.v3.MetadataMatcher metadata = 7; - // Negates matching the provided principal. For instance, if the value of `not_id` would match, - // this principal would not match. Conversely, if the value of `not_id` would not match, this - // principal would match. + // Negates matching the provided principal. For instance, if the value of + // `not_id` would match, this principal would not match. Conversely, if the + // value of `not_id` would not match, this principal would match. Principal not_id = 8; } } diff --git a/api/envoy/config/rbac/v4alpha/rbac.proto b/api/envoy/config/rbac/v4alpha/rbac.proto index 11b69b16e679..cc9d8933abab 100644 --- a/api/envoy/config/rbac/v4alpha/rbac.proto +++ b/api/envoy/config/rbac/v4alpha/rbac.proto @@ -23,8 +23,14 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: Role Based Access Control (RBAC)] // Role Based Access Control (RBAC) provides service-level and method-level access control for a -// service. RBAC policies are additive. The policies are examined in order. A request is allowed -// once a matching policy is found (suppose the `action` is ALLOW). +// service. RBAC policies are additive. The policies are examined in order. Requests are allowed +// or denied based on the `action` and whether a matching policy is found. For instance, if the +// action is ALLOW and a matching policy is found the request should be allowed. +// +// RBAC can also be used to make access logging decisions by communicating with access loggers +// through dynamic metadata. When the action is LOG and at least one policy matches, the +// `access_log_hint` value in the shared key namespace 'envoy.common' is set to `true` indicating +// the request should be logged. // // Here is an example of RBAC configuration. It has two policies: // @@ -67,39 +73,55 @@ message RBAC { // Should we do safe-list or block-list style access control? enum Action { - // The policies grant access to principals. The rest is denied. This is safe-list style + // The policies grant access to principals. The rest are denied. This is safe-list style // access control. This is the default type. ALLOW = 0; - // The policies deny access to principals. The rest is allowed. This is block-list style + // The policies deny access to principals. The rest are allowed. This is block-list style // access control. DENY = 1; + + // The policies set the `access_log_hint` dynamic metadata key based on if requests match. + // All requests are allowed. + LOG = 2; } - // The action to take if a policy matches. The request is allowed if and only if: + // The action to take if a policy matches. Every action either allows or denies a request, + // and can also carry out action-specific operations. + // + // Actions: + // + // * ALLOW: Allows the request if and only if there is a policy that matches + // the request. + // * DENY: Allows the request if and only if there are no policies that + // match the request. + // * LOG: Allows all requests. If at least one policy matches, the dynamic + // metadata key `access_log_hint` is set to the value `true` under the shared + // key namespace 'envoy.common'. If no policies match, it is set to `false`. + // Other actions do not modify this key. // - // * `action` is "ALLOWED" and at least one policy matches - // * `action` is "DENY" and none of the policies match Action action = 1; // Maps from policy name to policy. A match occurs when at least one policy matches the request. map policies = 2; } -// Policy specifies a role and the principals that are assigned/denied the role. A policy matches if -// and only if at least one of its permissions match the action taking place AND at least one of its -// principals match the downstream AND the condition is true if specified. +// Policy specifies a role and the principals that are assigned/denied the role. +// A policy matches if and only if at least one of its permissions match the +// action taking place AND at least one of its principals match the downstream +// AND the condition is true if specified. message Policy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v3.Policy"; - // Required. The set of permissions that define a role. Each permission is matched with OR - // semantics. To match all actions for this policy, a single Permission with the `any` field set - // to true should be used. + // Required. The set of permissions that define a role. Each permission is + // matched with OR semantics. To match all actions for this policy, a single + // Permission with the `any` field set to true should be used. repeated Permission permissions = 1 [(validate.rules).repeated = {min_items: 1}]; - // Required. The set of principals that are assigned/denied the role based on “action”. Each - // principal is matched with OR semantics. To match all downstreams for this policy, a single - // Principal with the `any` field set to true should be used. + // Required. The set of principals that are assigned/denied the role based on + // “action”. Each principal is matched with OR semantics. To match all + // downstreams for this policy, a single Principal with the `any` field set to + // true should be used. repeated Principal principals = 2 [(validate.rules).repeated = {min_items: 1}]; oneof expression_specifier { @@ -160,9 +182,9 @@ message Permission { // Metadata that describes additional information about the action. type.matcher.v4alpha.MetadataMatcher metadata = 7; - // Negates matching the provided permission. For instance, if the value of `not_rule` would - // match, this permission would not match. Conversely, if the value of `not_rule` would not - // match, this permission would match. + // Negates matching the provided permission. For instance, if the value of + // `not_rule` would match, this permission would not match. Conversely, if + // the value of `not_rule` would not match, this permission would match. Permission not_rule = 8; // The request server from the client's connection request. This is @@ -175,7 +197,8 @@ message Permission { // // * If the :ref:`TLS Inspector ` // filter is not added, and if a `FilterChainMatch` is not defined for - // the :ref:`server name `, + // the :ref:`server name + // `, // a TLS connection's requested SNI server name will be treated as if it // wasn't present. // @@ -188,13 +211,14 @@ message Permission { } } -// Principal defines an identity or a group of identities for a downstream subject. +// Principal defines an identity or a group of identities for a downstream +// subject. // [#next-free-field: 12] message Principal { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v3.Principal"; - // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. Depending on the context, - // each are applied with the associated behavior. + // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. + // Depending on the context, each are applied with the associated behavior. message Set { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v3.Principal.Set"; @@ -209,9 +233,9 @@ message Principal { reserved 1; - // The name of the principal. If set, The URI SAN or DNS SAN in that order is used from the - // certificate, otherwise the subject field is used. If unset, it applies to any user that is - // authenticated. + // The name of the principal. If set, The URI SAN or DNS SAN in that order + // is used from the certificate, otherwise the subject field is used. If + // unset, it applies to any user that is authenticated. type.matcher.v4alpha.StringMatcher principal_name = 2; } @@ -222,10 +246,12 @@ message Principal { oneof identifier { option (validate.required) = true; - // A set of identifiers that all must match in order to define the downstream. + // A set of identifiers that all must match in order to define the + // downstream. Set and_ids = 1; - // A set of identifiers at least one must match in order to define the downstream. + // A set of identifiers at least one must match in order to define the + // downstream. Set or_ids = 2; // When any is set, it matches any downstream. @@ -236,21 +262,23 @@ message Principal { // A CIDR block that describes the downstream remote/origin address. // Note: This is always the physical peer even if the - // :ref:`remote_ip ` is inferred - // from for example the x-forwarder-for header, proxy protocol, etc. + // :ref:`remote_ip ` is + // inferred from for example the x-forwarder-for header, proxy protocol, + // etc. core.v4alpha.CidrRange direct_remote_ip = 10; // A CIDR block that describes the downstream remote/origin address. // Note: This may not be the physical peer and could be different from the - // :ref:`direct_remote_ip `. - // E.g, if the remote ip is inferred from for example the x-forwarder-for header, - // proxy protocol, etc. + // :ref:`direct_remote_ip + // `. E.g, if the + // remote ip is inferred from for example the x-forwarder-for header, proxy + // protocol, etc. core.v4alpha.CidrRange remote_ip = 11; - // A header (or pseudo-header such as :path or :method) on the incoming HTTP request. Only - // available for HTTP request. - // Note: the pseudo-header :path includes the query and fragment string. Use the `url_path` - // field if you want to match the URL path without the query and fragment string. + // A header (or pseudo-header such as :path or :method) on the incoming HTTP + // request. Only available for HTTP request. Note: the pseudo-header :path + // includes the query and fragment string. Use the `url_path` field if you + // want to match the URL path without the query and fragment string. route.v4alpha.HeaderMatcher header = 6; // A URL path on the incoming HTTP request. Only available for HTTP. @@ -259,9 +287,9 @@ message Principal { // Metadata that describes additional information about the principal. type.matcher.v4alpha.MetadataMatcher metadata = 7; - // Negates matching the provided principal. For instance, if the value of `not_id` would match, - // this principal would not match. Conversely, if the value of `not_id` would not match, this - // principal would match. + // Negates matching the provided principal. For instance, if the value of + // `not_id` would match, this principal would not match. Conversely, if the + // value of `not_id` would not match, this principal would match. Principal not_id = 8; } } diff --git a/docs/root/configuration/advanced/well_known_dynamic_metadata.rst b/docs/root/configuration/advanced/well_known_dynamic_metadata.rst index 0088a85d9b94..4d7f8ed3872c 100644 --- a/docs/root/configuration/advanced/well_known_dynamic_metadata.rst +++ b/docs/root/configuration/advanced/well_known_dynamic_metadata.rst @@ -27,3 +27,24 @@ The following Envoy filters can be configured to consume dynamic metadata emitte * :ref:`External Authorization Filter via the metadata context namespaces ` * :ref:`RateLimit Filter limit override ` + +.. _shared_dynamic_metadata: + +Shared Dynamic Metadata +----------------------- +Dynamic metadata that is set by multiple filters is placed in the common key namespace `envoy.common`. Refer to the corresponding rules when setting this metadata. + +.. csv-table:: + :header: Name, Type, Description, Rules + :widths: 1, 1, 3, 3 + + access_log_hint, boolean, Whether access loggers should log the request., "When this metadata is already set: A `true` value should not be overwritten by a `false` value, while a `false` value can be overwritten by a `true` value." + +The following Envoy filters emit shared dynamic metadata. + +* :ref:`Role Based Access Control (RBAC) Filter ` +* :ref:`Role Based Access Control (RBAC) Network Filter ` + +The following filters consume shared dynamic metadata. + +* :ref:`Metadata Access Log Filter` diff --git a/docs/root/configuration/http/http_filters/rbac_filter.rst b/docs/root/configuration/http/http_filters/rbac_filter.rst index d6068bbdcc6c..5db112d924ef 100644 --- a/docs/root/configuration/http/http_filters/rbac_filter.rst +++ b/docs/root/configuration/http/http_filters/rbac_filter.rst @@ -36,6 +36,8 @@ owning HTTP connection manager. denied, Counter, Total requests that were denied access shadow_allowed, Counter, Total requests that would be allowed access by the filter's shadow rules shadow_denied, Counter, Total requests that would be denied access by the filter's shadow rules + logged, Counter, Total requests that should be logged + not_logged, Counter, Total requests that should not be logged .. _config_http_filters_rbac_dynamic_metadata: @@ -50,3 +52,4 @@ The RBAC filter emits the following dynamic metadata. shadow_effective_policy_id, string, The effective shadow policy ID matching the action (if any). shadow_engine_result, string, The engine result for the shadow rules (i.e. either `allowed` or `denied`). + access_log_hint, boolean, Whether the request should be logged. This metadata is shared and set under the key namespace 'envoy.common' (See :ref:`Shared Dynamic Metadata`). diff --git a/docs/root/configuration/listeners/network_filters/rbac_filter.rst b/docs/root/configuration/listeners/network_filters/rbac_filter.rst index d07417492045..68ae9f2172d4 100644 --- a/docs/root/configuration/listeners/network_filters/rbac_filter.rst +++ b/docs/root/configuration/listeners/network_filters/rbac_filter.rst @@ -26,6 +26,8 @@ The RBAC network filter outputs statistics in the *.rbac.* namespac denied, Counter, Total requests that were denied access shadow_allowed, Counter, Total requests that would be allowed access by the filter's shadow rules shadow_denied, Counter, Total requests that would be denied access by the filter's shadow rules + logged, Counter, Total requests that should be logged + not_logged, Counter, Total requests that should not be logged .. _config_network_filters_rbac_dynamic_metadata: @@ -40,3 +42,4 @@ The RBAC filter emits the following dynamic metadata. shadow_effective_policy_id, string, The effective shadow policy ID matching the action (if any). shadow_engine_result, string, The engine result for the shadow rules (i.e. either `allowed` or `denied`). + access_log_hint, boolean, Whether the request should be logged. This metadata is shared and set under the key namespace 'envoy.common' (See :ref:`Shared Dynamic Metadata`). diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index f5b6217a1757..4a6ac04a2576 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -60,6 +60,7 @@ New Features * lua: added Lua APIs to access :ref:`SSL connection info ` object. * postgres network filter: :ref:`metadata ` is produced based on SQL query. * ratelimit: added :ref:`enable_x_ratelimit_headers ` option to enable `X-RateLimit-*` headers as defined in `draft RFC `_. +* rbac filter: added a log action to the :ref:`RBAC filter ` which sets dynamic metadata to inform access loggers whether to log. * router: added new :ref:`envoy-ratelimited` retry policy, which allows retrying envoy's own rate limited responses. diff --git a/generated_api_shadow/envoy/config/rbac/v3/rbac.proto b/generated_api_shadow/envoy/config/rbac/v3/rbac.proto index 10520b1ba38f..278e6857603f 100644 --- a/generated_api_shadow/envoy/config/rbac/v3/rbac.proto +++ b/generated_api_shadow/envoy/config/rbac/v3/rbac.proto @@ -24,8 +24,14 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Role Based Access Control (RBAC)] // Role Based Access Control (RBAC) provides service-level and method-level access control for a -// service. RBAC policies are additive. The policies are examined in order. A request is allowed -// once a matching policy is found (suppose the `action` is ALLOW). +// service. RBAC policies are additive. The policies are examined in order. Requests are allowed +// or denied based on the `action` and whether a matching policy is found. For instance, if the +// action is ALLOW and a matching policy is found the request should be allowed. +// +// RBAC can also be used to make access logging decisions by communicating with access loggers +// through dynamic metadata. When the action is LOG and at least one policy matches, the +// `access_log_hint` value in the shared key namespace 'envoy.common' is set to `true` indicating +// the request should be logged. // // Here is an example of RBAC configuration. It has two policies: // @@ -68,39 +74,55 @@ message RBAC { // Should we do safe-list or block-list style access control? enum Action { - // The policies grant access to principals. The rest is denied. This is safe-list style + // The policies grant access to principals. The rest are denied. This is safe-list style // access control. This is the default type. ALLOW = 0; - // The policies deny access to principals. The rest is allowed. This is block-list style + // The policies deny access to principals. The rest are allowed. This is block-list style // access control. DENY = 1; + + // The policies set the `access_log_hint` dynamic metadata key based on if requests match. + // All requests are allowed. + LOG = 2; } - // The action to take if a policy matches. The request is allowed if and only if: + // The action to take if a policy matches. Every action either allows or denies a request, + // and can also carry out action-specific operations. + // + // Actions: + // + // * ALLOW: Allows the request if and only if there is a policy that matches + // the request. + // * DENY: Allows the request if and only if there are no policies that + // match the request. + // * LOG: Allows all requests. If at least one policy matches, the dynamic + // metadata key `access_log_hint` is set to the value `true` under the shared + // key namespace 'envoy.common'. If no policies match, it is set to `false`. + // Other actions do not modify this key. // - // * `action` is "ALLOWED" and at least one policy matches - // * `action` is "DENY" and none of the policies match Action action = 1; // Maps from policy name to policy. A match occurs when at least one policy matches the request. map policies = 2; } -// Policy specifies a role and the principals that are assigned/denied the role. A policy matches if -// and only if at least one of its permissions match the action taking place AND at least one of its -// principals match the downstream AND the condition is true if specified. +// Policy specifies a role and the principals that are assigned/denied the role. +// A policy matches if and only if at least one of its permissions match the +// action taking place AND at least one of its principals match the downstream +// AND the condition is true if specified. message Policy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v2.Policy"; - // Required. The set of permissions that define a role. Each permission is matched with OR - // semantics. To match all actions for this policy, a single Permission with the `any` field set - // to true should be used. + // Required. The set of permissions that define a role. Each permission is + // matched with OR semantics. To match all actions for this policy, a single + // Permission with the `any` field set to true should be used. repeated Permission permissions = 1 [(validate.rules).repeated = {min_items: 1}]; - // Required. The set of principals that are assigned/denied the role based on “action”. Each - // principal is matched with OR semantics. To match all downstreams for this policy, a single - // Principal with the `any` field set to true should be used. + // Required. The set of principals that are assigned/denied the role based on + // “action”. Each principal is matched with OR semantics. To match all + // downstreams for this policy, a single Principal with the `any` field set to + // true should be used. repeated Principal principals = 2 [(validate.rules).repeated = {min_items: 1}]; // An optional symbolic expression specifying an access control @@ -161,9 +183,9 @@ message Permission { // Metadata that describes additional information about the action. type.matcher.v3.MetadataMatcher metadata = 7; - // Negates matching the provided permission. For instance, if the value of `not_rule` would - // match, this permission would not match. Conversely, if the value of `not_rule` would not - // match, this permission would match. + // Negates matching the provided permission. For instance, if the value of + // `not_rule` would match, this permission would not match. Conversely, if + // the value of `not_rule` would not match, this permission would match. Permission not_rule = 8; // The request server from the client's connection request. This is @@ -176,7 +198,8 @@ message Permission { // // * If the :ref:`TLS Inspector ` // filter is not added, and if a `FilterChainMatch` is not defined for - // the :ref:`server name `, + // the :ref:`server name + // `, // a TLS connection's requested SNI server name will be treated as if it // wasn't present. // @@ -189,13 +212,14 @@ message Permission { } } -// Principal defines an identity or a group of identities for a downstream subject. +// Principal defines an identity or a group of identities for a downstream +// subject. // [#next-free-field: 12] message Principal { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v2.Principal"; - // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. Depending on the context, - // each are applied with the associated behavior. + // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. + // Depending on the context, each are applied with the associated behavior. message Set { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v2.Principal.Set"; @@ -210,19 +234,21 @@ message Principal { reserved 1; - // The name of the principal. If set, The URI SAN or DNS SAN in that order is used from the - // certificate, otherwise the subject field is used. If unset, it applies to any user that is - // authenticated. + // The name of the principal. If set, The URI SAN or DNS SAN in that order + // is used from the certificate, otherwise the subject field is used. If + // unset, it applies to any user that is authenticated. type.matcher.v3.StringMatcher principal_name = 2; } oneof identifier { option (validate.required) = true; - // A set of identifiers that all must match in order to define the downstream. + // A set of identifiers that all must match in order to define the + // downstream. Set and_ids = 1; - // A set of identifiers at least one must match in order to define the downstream. + // A set of identifiers at least one must match in order to define the + // downstream. Set or_ids = 2; // When any is set, it matches any downstream. @@ -237,21 +263,23 @@ message Principal { // A CIDR block that describes the downstream remote/origin address. // Note: This is always the physical peer even if the - // :ref:`remote_ip ` is inferred - // from for example the x-forwarder-for header, proxy protocol, etc. + // :ref:`remote_ip ` is + // inferred from for example the x-forwarder-for header, proxy protocol, + // etc. core.v3.CidrRange direct_remote_ip = 10; // A CIDR block that describes the downstream remote/origin address. // Note: This may not be the physical peer and could be different from the - // :ref:`direct_remote_ip `. - // E.g, if the remote ip is inferred from for example the x-forwarder-for header, - // proxy protocol, etc. + // :ref:`direct_remote_ip + // `. E.g, if the + // remote ip is inferred from for example the x-forwarder-for header, proxy + // protocol, etc. core.v3.CidrRange remote_ip = 11; - // A header (or pseudo-header such as :path or :method) on the incoming HTTP request. Only - // available for HTTP request. - // Note: the pseudo-header :path includes the query and fragment string. Use the `url_path` - // field if you want to match the URL path without the query and fragment string. + // A header (or pseudo-header such as :path or :method) on the incoming HTTP + // request. Only available for HTTP request. Note: the pseudo-header :path + // includes the query and fragment string. Use the `url_path` field if you + // want to match the URL path without the query and fragment string. route.v3.HeaderMatcher header = 6; // A URL path on the incoming HTTP request. Only available for HTTP. @@ -260,9 +288,9 @@ message Principal { // Metadata that describes additional information about the principal. type.matcher.v3.MetadataMatcher metadata = 7; - // Negates matching the provided principal. For instance, if the value of `not_id` would match, - // this principal would not match. Conversely, if the value of `not_id` would not match, this - // principal would match. + // Negates matching the provided principal. For instance, if the value of + // `not_id` would match, this principal would not match. Conversely, if the + // value of `not_id` would not match, this principal would match. Principal not_id = 8; } } diff --git a/generated_api_shadow/envoy/config/rbac/v4alpha/rbac.proto b/generated_api_shadow/envoy/config/rbac/v4alpha/rbac.proto index 3d8dae2402ea..7139dfaa1485 100644 --- a/generated_api_shadow/envoy/config/rbac/v4alpha/rbac.proto +++ b/generated_api_shadow/envoy/config/rbac/v4alpha/rbac.proto @@ -23,8 +23,14 @@ option (udpa.annotations.file_status).package_version_status = NEXT_MAJOR_VERSIO // [#protodoc-title: Role Based Access Control (RBAC)] // Role Based Access Control (RBAC) provides service-level and method-level access control for a -// service. RBAC policies are additive. The policies are examined in order. A request is allowed -// once a matching policy is found (suppose the `action` is ALLOW). +// service. RBAC policies are additive. The policies are examined in order. Requests are allowed +// or denied based on the `action` and whether a matching policy is found. For instance, if the +// action is ALLOW and a matching policy is found the request should be allowed. +// +// RBAC can also be used to make access logging decisions by communicating with access loggers +// through dynamic metadata. When the action is LOG and at least one policy matches, the +// `access_log_hint` value in the shared key namespace 'envoy.common' is set to `true` indicating +// the request should be logged. // // Here is an example of RBAC configuration. It has two policies: // @@ -67,39 +73,55 @@ message RBAC { // Should we do safe-list or block-list style access control? enum Action { - // The policies grant access to principals. The rest is denied. This is safe-list style + // The policies grant access to principals. The rest are denied. This is safe-list style // access control. This is the default type. ALLOW = 0; - // The policies deny access to principals. The rest is allowed. This is block-list style + // The policies deny access to principals. The rest are allowed. This is block-list style // access control. DENY = 1; + + // The policies set the `access_log_hint` dynamic metadata key based on if requests match. + // All requests are allowed. + LOG = 2; } - // The action to take if a policy matches. The request is allowed if and only if: + // The action to take if a policy matches. Every action either allows or denies a request, + // and can also carry out action-specific operations. + // + // Actions: + // + // * ALLOW: Allows the request if and only if there is a policy that matches + // the request. + // * DENY: Allows the request if and only if there are no policies that + // match the request. + // * LOG: Allows all requests. If at least one policy matches, the dynamic + // metadata key `access_log_hint` is set to the value `true` under the shared + // key namespace 'envoy.common'. If no policies match, it is set to `false`. + // Other actions do not modify this key. // - // * `action` is "ALLOWED" and at least one policy matches - // * `action` is "DENY" and none of the policies match Action action = 1; // Maps from policy name to policy. A match occurs when at least one policy matches the request. map policies = 2; } -// Policy specifies a role and the principals that are assigned/denied the role. A policy matches if -// and only if at least one of its permissions match the action taking place AND at least one of its -// principals match the downstream AND the condition is true if specified. +// Policy specifies a role and the principals that are assigned/denied the role. +// A policy matches if and only if at least one of its permissions match the +// action taking place AND at least one of its principals match the downstream +// AND the condition is true if specified. message Policy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v3.Policy"; - // Required. The set of permissions that define a role. Each permission is matched with OR - // semantics. To match all actions for this policy, a single Permission with the `any` field set - // to true should be used. + // Required. The set of permissions that define a role. Each permission is + // matched with OR semantics. To match all actions for this policy, a single + // Permission with the `any` field set to true should be used. repeated Permission permissions = 1 [(validate.rules).repeated = {min_items: 1}]; - // Required. The set of principals that are assigned/denied the role based on “action”. Each - // principal is matched with OR semantics. To match all downstreams for this policy, a single - // Principal with the `any` field set to true should be used. + // Required. The set of principals that are assigned/denied the role based on + // “action”. Each principal is matched with OR semantics. To match all + // downstreams for this policy, a single Principal with the `any` field set to + // true should be used. repeated Principal principals = 2 [(validate.rules).repeated = {min_items: 1}]; oneof expression_specifier { @@ -160,9 +182,9 @@ message Permission { // Metadata that describes additional information about the action. type.matcher.v4alpha.MetadataMatcher metadata = 7; - // Negates matching the provided permission. For instance, if the value of `not_rule` would - // match, this permission would not match. Conversely, if the value of `not_rule` would not - // match, this permission would match. + // Negates matching the provided permission. For instance, if the value of + // `not_rule` would match, this permission would not match. Conversely, if + // the value of `not_rule` would not match, this permission would match. Permission not_rule = 8; // The request server from the client's connection request. This is @@ -175,7 +197,8 @@ message Permission { // // * If the :ref:`TLS Inspector ` // filter is not added, and if a `FilterChainMatch` is not defined for - // the :ref:`server name `, + // the :ref:`server name + // `, // a TLS connection's requested SNI server name will be treated as if it // wasn't present. // @@ -188,13 +211,14 @@ message Permission { } } -// Principal defines an identity or a group of identities for a downstream subject. +// Principal defines an identity or a group of identities for a downstream +// subject. // [#next-free-field: 12] message Principal { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v3.Principal"; - // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. Depending on the context, - // each are applied with the associated behavior. + // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. + // Depending on the context, each are applied with the associated behavior. message Set { option (udpa.annotations.versioning).previous_message_type = "envoy.config.rbac.v3.Principal.Set"; @@ -209,19 +233,21 @@ message Principal { reserved 1; - // The name of the principal. If set, The URI SAN or DNS SAN in that order is used from the - // certificate, otherwise the subject field is used. If unset, it applies to any user that is - // authenticated. + // The name of the principal. If set, The URI SAN or DNS SAN in that order + // is used from the certificate, otherwise the subject field is used. If + // unset, it applies to any user that is authenticated. type.matcher.v4alpha.StringMatcher principal_name = 2; } oneof identifier { option (validate.required) = true; - // A set of identifiers that all must match in order to define the downstream. + // A set of identifiers that all must match in order to define the + // downstream. Set and_ids = 1; - // A set of identifiers at least one must match in order to define the downstream. + // A set of identifiers at least one must match in order to define the + // downstream. Set or_ids = 2; // When any is set, it matches any downstream. @@ -236,21 +262,23 @@ message Principal { // A CIDR block that describes the downstream remote/origin address. // Note: This is always the physical peer even if the - // :ref:`remote_ip ` is inferred - // from for example the x-forwarder-for header, proxy protocol, etc. + // :ref:`remote_ip ` is + // inferred from for example the x-forwarder-for header, proxy protocol, + // etc. core.v4alpha.CidrRange direct_remote_ip = 10; // A CIDR block that describes the downstream remote/origin address. // Note: This may not be the physical peer and could be different from the - // :ref:`direct_remote_ip `. - // E.g, if the remote ip is inferred from for example the x-forwarder-for header, - // proxy protocol, etc. + // :ref:`direct_remote_ip + // `. E.g, if the + // remote ip is inferred from for example the x-forwarder-for header, proxy + // protocol, etc. core.v4alpha.CidrRange remote_ip = 11; - // A header (or pseudo-header such as :path or :method) on the incoming HTTP request. Only - // available for HTTP request. - // Note: the pseudo-header :path includes the query and fragment string. Use the `url_path` - // field if you want to match the URL path without the query and fragment string. + // A header (or pseudo-header such as :path or :method) on the incoming HTTP + // request. Only available for HTTP request. Note: the pseudo-header :path + // includes the query and fragment string. Use the `url_path` field if you + // want to match the URL path without the query and fragment string. route.v4alpha.HeaderMatcher header = 6; // A URL path on the incoming HTTP request. Only available for HTTP. @@ -259,9 +287,9 @@ message Principal { // Metadata that describes additional information about the principal. type.matcher.v4alpha.MetadataMatcher metadata = 7; - // Negates matching the provided principal. For instance, if the value of `not_id` would match, - // this principal would not match. Conversely, if the value of `not_id` would not match, this - // principal would match. + // Negates matching the provided principal. For instance, if the value of + // `not_id` would match, this principal would not match. Conversely, if the + // value of `not_id` would not match, this principal would match. Principal not_id = 8; } } diff --git a/source/extensions/filters/common/rbac/engine.h b/source/extensions/filters/common/rbac/engine.h index a833867dd02a..7174d4edb860 100644 --- a/source/extensions/filters/common/rbac/engine.h +++ b/source/extensions/filters/common/rbac/engine.h @@ -19,32 +19,32 @@ class RoleBasedAccessControlEngine { virtual ~RoleBasedAccessControlEngine() = default; /** - * Returns whether or not the current action is permitted. + * Handles action-specific operations and returns whether or not the request is permitted. * * @param connection the downstream connection used to identify the action/principal. * @param headers the headers of the incoming request used to identify the action/principal. An * empty map should be used if there are no headers available. * @param info the per-request or per-connection stream info with additional information - * about the action/principal. + * about the action/principal. Can be modified by the LOG Action. * @param effective_policy_id it will be filled by the matching policy's ID, * which is used to identity the source of the allow/deny. */ - virtual bool allowed(const Network::Connection& connection, - const Envoy::Http::RequestHeaderMap& headers, - const StreamInfo::StreamInfo& info, - std::string* effective_policy_id) const PURE; + virtual bool handleAction(const Network::Connection& connection, + const Envoy::Http::RequestHeaderMap& headers, + StreamInfo::StreamInfo& info, + std::string* effective_policy_id) const PURE; /** - * Returns whether or not the current action is permitted. + * Handles action-specific operations and returns whether or not the request is permitted. * * @param connection the downstream connection used to identify the action/principal. * @param info the per-request or per-connection stream info with additional information - * about the action/principal. + * about the action/principal. Can be modified by the LOG Action. * @param effective_policy_id it will be filled by the matching policy's ID, * which is used to identity the source of the allow/deny. */ - virtual bool allowed(const Network::Connection& connection, const StreamInfo::StreamInfo& info, - std::string* effective_policy_id) const PURE; + virtual bool handleAction(const Network::Connection& connection, StreamInfo::StreamInfo& info, + std::string* effective_policy_id) const PURE; }; } // namespace RBAC diff --git a/source/extensions/filters/common/rbac/engine_impl.cc b/source/extensions/filters/common/rbac/engine_impl.cc index d9717ef509c0..dc2a6ba79222 100644 --- a/source/extensions/filters/common/rbac/engine_impl.cc +++ b/source/extensions/filters/common/rbac/engine_impl.cc @@ -11,8 +11,8 @@ namespace Common { namespace RBAC { RoleBasedAccessControlEngineImpl::RoleBasedAccessControlEngineImpl( - const envoy::config::rbac::v3::RBAC& rules) - : allowed_if_matched_(rules.action() == envoy::config::rbac::v3::RBAC::ALLOW) { + const envoy::config::rbac::v3::RBAC& rules, const EnforcementMode mode) + : action_(rules.action()), mode_(mode) { // guard expression builder by presence of a condition in policies for (const auto& policy : rules.policies()) { if (policy.second.has_condition()) { @@ -26,10 +26,43 @@ RoleBasedAccessControlEngineImpl::RoleBasedAccessControlEngineImpl( } } -bool RoleBasedAccessControlEngineImpl::allowed(const Network::Connection& connection, - const Envoy::Http::RequestHeaderMap& headers, - const StreamInfo::StreamInfo& info, - std::string* effective_policy_id) const { +bool RoleBasedAccessControlEngineImpl::handleAction(const Network::Connection& connection, + StreamInfo::StreamInfo& info, + std::string* effective_policy_id) const { + return handleAction(connection, *Http::StaticEmptyHeaders::get().request_headers, info, + effective_policy_id); +} + +bool RoleBasedAccessControlEngineImpl::handleAction(const Network::Connection& connection, + const Envoy::Http::RequestHeaderMap& headers, + StreamInfo::StreamInfo& info, + std::string* effective_policy_id) const { + bool matched = checkPolicyMatch(connection, info, headers, effective_policy_id); + + switch (action_) { + case envoy::config::rbac::v3::RBAC::ALLOW: + return matched; + case envoy::config::rbac::v3::RBAC::DENY: + return !matched; + case envoy::config::rbac::v3::RBAC::LOG: { + // If not shadow enforcement, set shared log metadata + if (mode_ != EnforcementMode::Shadow) { + ProtobufWkt::Struct log_metadata; + auto& log_fields = *log_metadata.mutable_fields(); + log_fields[DynamicMetadataKeysSingleton::get().AccessLogKey].set_bool_value(matched); + info.setDynamicMetadata(DynamicMetadataKeysSingleton::get().CommonNamespace, log_metadata); + } + + return true; + } + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +bool RoleBasedAccessControlEngineImpl::checkPolicyMatch( + const Network::Connection& connection, const StreamInfo::StreamInfo& info, + const Envoy::Http::RequestHeaderMap& headers, std::string* effective_policy_id) const { bool matched = false; for (const auto& policy : policies_) { @@ -42,17 +75,7 @@ bool RoleBasedAccessControlEngineImpl::allowed(const Network::Connection& connec } } - // only allowed if: - // - matched and ALLOW action - // - not matched and DENY action - return matched == allowed_if_matched_; -} - -bool RoleBasedAccessControlEngineImpl::allowed(const Network::Connection& connection, - const StreamInfo::StreamInfo& info, - std::string* effective_policy_id) const { - return allowed(connection, *Http::StaticEmptyHeaders::get().request_headers, info, - effective_policy_id); + return matched; } } // namespace RBAC diff --git a/source/extensions/filters/common/rbac/engine_impl.h b/source/extensions/filters/common/rbac/engine_impl.h index 261b45b0aa13..0aacfb41f8e1 100644 --- a/source/extensions/filters/common/rbac/engine_impl.h +++ b/source/extensions/filters/common/rbac/engine_impl.h @@ -11,18 +11,40 @@ namespace Filters { namespace Common { namespace RBAC { +class DynamicMetadataKeys { +public: + const std::string ShadowEffectivePolicyIdField{"shadow_effective_policy_id"}; + const std::string ShadowEngineResultField{"shadow_engine_result"}; + const std::string EngineResultAllowed{"allowed"}; + const std::string EngineResultDenied{"denied"}; + const std::string AccessLogKey{"access_log_hint"}; + const std::string CommonNamespace{"envoy.common"}; +}; + +using DynamicMetadataKeysSingleton = ConstSingleton; + +enum class EnforcementMode { Enforced, Shadow }; + class RoleBasedAccessControlEngineImpl : public RoleBasedAccessControlEngine, NonCopyable { public: - RoleBasedAccessControlEngineImpl(const envoy::config::rbac::v3::RBAC& rules); + RoleBasedAccessControlEngineImpl(const envoy::config::rbac::v3::RBAC& rules, + const EnforcementMode mode = EnforcementMode::Enforced); - bool allowed(const Network::Connection& connection, const Envoy::Http::RequestHeaderMap& headers, - const StreamInfo::StreamInfo& info, std::string* effective_policy_id) const override; + bool handleAction(const Network::Connection& connection, + const Envoy::Http::RequestHeaderMap& headers, StreamInfo::StreamInfo& info, + std::string* effective_policy_id) const override; - bool allowed(const Network::Connection& connection, const StreamInfo::StreamInfo& info, - std::string* effective_policy_id) const override; + bool handleAction(const Network::Connection& connection, StreamInfo::StreamInfo& info, + std::string* effective_policy_id) const override; private: - const bool allowed_if_matched_; + // Checks whether the request matches any policies + bool checkPolicyMatch(const Network::Connection& connection, const StreamInfo::StreamInfo& info, + const Envoy::Http::RequestHeaderMap& headers, + std::string* effective_policy_id) const; + + const envoy::config::rbac::v3::RBAC::Action action_; + const EnforcementMode mode_; std::map> policies_; diff --git a/source/extensions/filters/common/rbac/utility.h b/source/extensions/filters/common/rbac/utility.h index a48efb813234..04635eb37411 100644 --- a/source/extensions/filters/common/rbac/utility.h +++ b/source/extensions/filters/common/rbac/utility.h @@ -12,16 +12,6 @@ namespace Filters { namespace Common { namespace RBAC { -class DynamicMetadataKeys { -public: - const std::string ShadowEffectivePolicyIdField{"shadow_effective_policy_id"}; - const std::string ShadowEngineResultField{"shadow_engine_result"}; - const std::string EngineResultAllowed{"allowed"}; - const std::string EngineResultDenied{"denied"}; -}; - -using DynamicMetadataKeysSingleton = ConstSingleton; - /** * All stats for the RBAC filter. @see stats_macros.h */ @@ -40,19 +30,18 @@ struct RoleBasedAccessControlFilterStats { RoleBasedAccessControlFilterStats generateStats(const std::string& prefix, Stats::Scope& scope); -enum class EnforcementMode { Enforced, Shadow }; - template std::unique_ptr createEngine(const ConfigType& config) { - return config.has_rules() ? std::make_unique(config.rules()) + return config.has_rules() ? std::make_unique( + config.rules(), EnforcementMode::Enforced) : nullptr; } template std::unique_ptr createShadowEngine(const ConfigType& config) { - return config.has_shadow_rules() - ? std::make_unique(config.shadow_rules()) - : nullptr; + return config.has_shadow_rules() ? std::make_unique( + config.shadow_rules(), EnforcementMode::Shadow) + : nullptr; } } // namespace RBAC diff --git a/source/extensions/filters/http/rbac/rbac_filter.cc b/source/extensions/filters/http/rbac/rbac_filter.cc index 6e1ff3ea3318..d396db7f52bc 100644 --- a/source/extensions/filters/http/rbac/rbac_filter.cc +++ b/source/extensions/filters/http/rbac/rbac_filter.cc @@ -80,8 +80,8 @@ RoleBasedAccessControlFilter::decodeHeaders(Http::RequestHeaderMap& headers, boo if (shadow_engine != nullptr) { std::string shadow_resp_code = Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().EngineResultAllowed; - if (shadow_engine->allowed(*callbacks_->connection(), headers, callbacks_->streamInfo(), - &effective_policy_id)) { + if (shadow_engine->handleAction(*callbacks_->connection(), headers, callbacks_->streamInfo(), + &effective_policy_id)) { ENVOY_LOG(debug, "shadow allowed"); config_->stats().shadow_allowed_.inc(); } else { @@ -109,7 +109,8 @@ RoleBasedAccessControlFilter::decodeHeaders(Http::RequestHeaderMap& headers, boo const auto engine = config_->engine(callbacks_->route(), Filters::Common::RBAC::EnforcementMode::Enforced); if (engine != nullptr) { - if (engine->allowed(*callbacks_->connection(), headers, callbacks_->streamInfo(), nullptr)) { + if (engine->handleAction(*callbacks_->connection(), headers, callbacks_->streamInfo(), + nullptr)) { ENVOY_LOG(debug, "enforced allowed"); config_->stats().allowed_.inc(); return Http::FilterHeadersStatus::Continue; diff --git a/source/extensions/filters/network/rbac/rbac_filter.cc b/source/extensions/filters/network/rbac/rbac_filter.cc index 1bc12017b3b6..3b328ed2815f 100644 --- a/source/extensions/filters/network/rbac/rbac_filter.cc +++ b/source/extensions/filters/network/rbac/rbac_filter.cc @@ -85,8 +85,10 @@ RoleBasedAccessControlFilter::checkEngine(Filters::Common::RBAC::EnforcementMode const auto engine = config_->engine(mode); if (engine != nullptr) { std::string effective_policy_id; - if (engine->allowed(callbacks_->connection(), callbacks_->connection().streamInfo(), - &effective_policy_id)) { + + // Check authorization decision and do Action operations + if (engine->handleAction(callbacks_->connection(), callbacks_->connection().streamInfo(), + &effective_policy_id)) { if (mode == Filters::Common::RBAC::EnforcementMode::Shadow) { ENVOY_LOG(debug, "shadow allowed"); config_->stats().shadow_allowed_.inc(); diff --git a/test/extensions/filters/common/rbac/engine_impl_test.cc b/test/extensions/filters/common/rbac/engine_impl_test.cc index 8f4f3d7e6ad1..b9d8608a9208 100644 --- a/test/extensions/filters/common/rbac/engine_impl_test.cc +++ b/test/extensions/filters/common/rbac/engine_impl_test.cc @@ -24,21 +24,56 @@ namespace Common { namespace RBAC { namespace { +enum class LogResult { Yes, No, Undecided }; + void checkEngine( - const RBAC::RoleBasedAccessControlEngineImpl& engine, bool expected, + RBAC::RoleBasedAccessControlEngineImpl& engine, bool expected, LogResult expected_log, + StreamInfo::StreamInfo& info, const Envoy::Network::Connection& connection = Envoy::Network::MockConnection(), - const Envoy::Http::RequestHeaderMap& headers = Envoy::Http::TestRequestHeaderMapImpl(), - const StreamInfo::StreamInfo& info = NiceMock()) { - EXPECT_EQ(expected, engine.allowed(connection, headers, info, nullptr)); + const Envoy::Http::RequestHeaderMap& headers = Envoy::Http::TestRequestHeaderMapImpl()) { + + bool engineRes = engine.handleAction(connection, headers, info, nullptr); + EXPECT_EQ(expected, engineRes); + + if (expected_log != LogResult::Undecided) { + auto filter_meta = info.dynamicMetadata().filter_metadata().at( + RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace); + EXPECT_EQ(expected_log == LogResult::Yes, + filter_meta.fields() + .at(RBAC::DynamicMetadataKeysSingleton::get().AccessLogKey) + .bool_value()); + } else { + EXPECT_EQ(info.dynamicMetadata().filter_metadata().end(), + info.dynamicMetadata().filter_metadata().find( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace)); + } +} + +void checkEngine( + RBAC::RoleBasedAccessControlEngineImpl& engine, bool expected, LogResult expected_log, + const Envoy::Network::Connection& connection = Envoy::Network::MockConnection(), + const Envoy::Http::RequestHeaderMap& headers = Envoy::Http::TestRequestHeaderMapImpl()) { + + NiceMock empty_info; + checkEngine(engine, expected, expected_log, empty_info, connection, headers); +} + +void onMetadata(NiceMock& info) { + ON_CALL(info, setDynamicMetadata("envoy.common", _)) + .WillByDefault(Invoke([&info](const std::string&, const ProtobufWkt::Struct& obj) { + (*info.metadata_.mutable_filter_metadata())["envoy.common"] = obj; + })); } TEST(RoleBasedAccessControlEngineImpl, Disabled) { envoy::config::rbac::v3::RBAC rbac; rbac.set_action(envoy::config::rbac::v3::RBAC::ALLOW); - checkEngine(RBAC::RoleBasedAccessControlEngineImpl(rbac), false); + RBAC::RoleBasedAccessControlEngineImpl engine_allow(rbac); + checkEngine(engine_allow, false, LogResult::Undecided); rbac.set_action(envoy::config::rbac::v3::RBAC::DENY); - checkEngine(RBAC::RoleBasedAccessControlEngineImpl(rbac), true); + RBAC::RoleBasedAccessControlEngineImpl engine_deny(rbac); + checkEngine(engine_deny, true, LogResult::Undecided); } // Test various invalid policies to validate the fix for @@ -143,11 +178,11 @@ TEST(RoleBasedAccessControlEngineImpl, AllowedAllowlist) { Envoy::Network::Address::InstanceConstSharedPtr addr = Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 123, false); EXPECT_CALL(Const(info), downstreamLocalAddress()).WillOnce(ReturnRef(addr)); - checkEngine(engine, true, conn, headers, info); + checkEngine(engine, true, LogResult::Undecided, info, conn, headers); addr = Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 456, false); EXPECT_CALL(Const(info), downstreamLocalAddress()).WillOnce(ReturnRef(addr)); - checkEngine(engine, false, conn, headers, info); + checkEngine(engine, false, LogResult::Undecided, info, conn, headers); } TEST(RoleBasedAccessControlEngineImpl, DeniedDenylist) { @@ -166,11 +201,11 @@ TEST(RoleBasedAccessControlEngineImpl, DeniedDenylist) { Envoy::Network::Address::InstanceConstSharedPtr addr = Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 123, false); EXPECT_CALL(Const(info), downstreamLocalAddress()).WillOnce(ReturnRef(addr)); - checkEngine(engine, false, conn, headers, info); + checkEngine(engine, false, LogResult::Undecided, info, conn, headers); addr = Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 456, false); EXPECT_CALL(Const(info), downstreamLocalAddress()).WillOnce(ReturnRef(addr)); - checkEngine(engine, true, conn, headers, info); + checkEngine(engine, true, LogResult::Undecided, info, conn, headers); } TEST(RoleBasedAccessControlEngineImpl, BasicCondition) { @@ -187,7 +222,7 @@ TEST(RoleBasedAccessControlEngineImpl, BasicCondition) { rbac.set_action(envoy::config::rbac::v3::RBAC::ALLOW); (*rbac.mutable_policies())["foo"] = policy; RBAC::RoleBasedAccessControlEngineImpl engine(rbac); - checkEngine(engine, false); + checkEngine(engine, false, LogResult::Undecided); } TEST(RoleBasedAccessControlEngineImpl, MalformedCondition) { @@ -209,6 +244,10 @@ TEST(RoleBasedAccessControlEngineImpl, MalformedCondition) { EXPECT_THROW_WITH_REGEX(RBAC::RoleBasedAccessControlEngineImpl engine(rbac), EnvoyException, "failed to create an expression: .*"); + + rbac.set_action(envoy::config::rbac::v3::RBAC::LOG); + EXPECT_THROW_WITH_REGEX(RBAC::RoleBasedAccessControlEngineImpl engine_log(rbac), EnvoyException, + "failed to create an expression: .*"); } TEST(RoleBasedAccessControlEngineImpl, MistypedCondition) { @@ -225,7 +264,7 @@ TEST(RoleBasedAccessControlEngineImpl, MistypedCondition) { rbac.set_action(envoy::config::rbac::v3::RBAC::ALLOW); (*rbac.mutable_policies())["foo"] = policy; RBAC::RoleBasedAccessControlEngineImpl engine(rbac); - checkEngine(engine, false); + checkEngine(engine, false, LogResult::Undecided); } TEST(RoleBasedAccessControlEngineImpl, ErrorCondition) { @@ -250,7 +289,7 @@ TEST(RoleBasedAccessControlEngineImpl, ErrorCondition) { rbac.set_action(envoy::config::rbac::v3::RBAC::ALLOW); (*rbac.mutable_policies())["foo"] = policy; RBAC::RoleBasedAccessControlEngineImpl engine(rbac); - checkEngine(engine, false, Envoy::Network::MockConnection()); + checkEngine(engine, false, LogResult::Undecided, Envoy::Network::MockConnection()); } TEST(RoleBasedAccessControlEngineImpl, HeaderCondition) { @@ -286,7 +325,7 @@ TEST(RoleBasedAccessControlEngineImpl, HeaderCondition) { std::string value = "bar"; headers.setReference(key, value); - checkEngine(engine, true, Envoy::Network::MockConnection(), headers); + checkEngine(engine, true, LogResult::Undecided, Envoy::Network::MockConnection(), headers); } TEST(RoleBasedAccessControlEngineImpl, MetadataCondition) { @@ -331,7 +370,7 @@ TEST(RoleBasedAccessControlEngineImpl, MetadataCondition) { Protobuf::MapPair("other", label)); EXPECT_CALL(Const(info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); - checkEngine(engine, true, Envoy::Network::MockConnection(), headers, info); + checkEngine(engine, true, LogResult::Undecided, info, Envoy::Network::MockConnection(), headers); } TEST(RoleBasedAccessControlEngineImpl, ConjunctiveCondition) { @@ -354,8 +393,44 @@ TEST(RoleBasedAccessControlEngineImpl, ConjunctiveCondition) { NiceMock info; Envoy::Network::Address::InstanceConstSharedPtr addr = Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 123, false); + EXPECT_CALL(Const(info), downstreamLocalAddress()).Times(1).WillRepeatedly(ReturnRef(addr)); + checkEngine(engine, false, LogResult::Undecided, info, conn, headers); +} + +// Log tests +TEST(RoleBasedAccessControlEngineImpl, DisabledLog) { + NiceMock info; + onMetadata(info); + + envoy::config::rbac::v3::RBAC rbac; + rbac.set_action(envoy::config::rbac::v3::RBAC::LOG); + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + checkEngine(engine, true, RBAC::LogResult::No, info); +} + +TEST(RoleBasedAccessControlEngineImpl, LogIfMatched) { + envoy::config::rbac::v3::Policy policy; + policy.add_permissions()->set_destination_port(123); + policy.add_principals()->set_any(true); + + envoy::config::rbac::v3::RBAC rbac; + rbac.set_action(envoy::config::rbac::v3::RBAC::LOG); + (*rbac.mutable_policies())["foo"] = policy; + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + + Envoy::Network::MockConnection conn; + Envoy::Http::TestRequestHeaderMapImpl headers; + NiceMock info; + onMetadata(info); + + Envoy::Network::Address::InstanceConstSharedPtr addr = + Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 123, false); + EXPECT_CALL(Const(info), downstreamLocalAddress()).WillOnce(ReturnRef(addr)); + checkEngine(engine, true, RBAC::LogResult::Yes, info, conn, headers); + + addr = Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 456, false); EXPECT_CALL(Const(info), downstreamLocalAddress()).WillOnce(ReturnRef(addr)); - checkEngine(engine, false, conn, headers, info); + checkEngine(engine, true, RBAC::LogResult::No, info, conn, headers); } } // namespace diff --git a/test/extensions/filters/common/rbac/mocks.h b/test/extensions/filters/common/rbac/mocks.h index fda95244c893..a99e97aa9ea6 100644 --- a/test/extensions/filters/common/rbac/mocks.h +++ b/test/extensions/filters/common/rbac/mocks.h @@ -14,16 +14,17 @@ namespace RBAC { class MockEngine : public RoleBasedAccessControlEngineImpl { public: - MockEngine(const envoy::config::rbac::v3::RBAC& rules) - : RoleBasedAccessControlEngineImpl(rules){}; + MockEngine(const envoy::config::rbac::v3::RBAC& rules, + const EnforcementMode mode = EnforcementMode::Enforced) + : RoleBasedAccessControlEngineImpl(rules, mode){}; - MOCK_METHOD(bool, allowed, + MOCK_METHOD(bool, handleAction, (const Envoy::Network::Connection&, const Envoy::Http::RequestHeaderMap&, - const StreamInfo::StreamInfo&, std::string* effective_policy_id), + StreamInfo::StreamInfo&, std::string* effective_policy_id), (const)); - MOCK_METHOD(bool, allowed, - (const Envoy::Network::Connection&, const StreamInfo::StreamInfo&, + MOCK_METHOD(bool, handleAction, + (const Envoy::Network::Connection&, StreamInfo::StreamInfo&, std::string* effective_policy_id), (const)); }; diff --git a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc index 28e4420db014..b7fcd3ebcbb7 100644 --- a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc +++ b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc @@ -64,6 +64,20 @@ name: rbac - any: true )EOF"; +const std::string RBAC_CONFIG_WITH_LOG_ACTION = R"EOF( +name: rbac +typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC + rules: + action: LOG + policies: + foo: + permissions: + - header: { name: ":method", exact_match: "GET" } + principals: + - any: true +)EOF"; + using RBACIntegrationTest = HttpProtocolIntegrationTest; INSTANTIATE_TEST_SUITE_P(Protocols, RBACIntegrationTest, @@ -277,5 +291,28 @@ TEST_P(RBACIntegrationTest, PathIgnoreCase) { } } +TEST_P(RBACIntegrationTest, LogConnectionAllow) { + config_helper_.addFilter(RBAC_CONFIG_WITH_LOG_ACTION); + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeRequestWithBody( + Http::TestRequestHeaderMapImpl{ + {":method", "POST"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "10.0.0.1"}, + }, + 1024); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); +} + } // namespace } // namespace Envoy diff --git a/test/extensions/filters/http/rbac/rbac_filter_test.cc b/test/extensions/filters/http/rbac/rbac_filter_test.cc index d445860f8394..519a49126bbb 100644 --- a/test/extensions/filters/http/rbac/rbac_filter_test.cc +++ b/test/extensions/filters/http/rbac/rbac_filter_test.cc @@ -24,9 +24,12 @@ namespace HttpFilters { namespace RBACFilter { namespace { +enum class LogResult { Yes, No, Undecided }; + class RoleBasedAccessControlFilterTest : public testing::Test { public: - RoleBasedAccessControlFilterConfigSharedPtr setupConfig() { + RoleBasedAccessControlFilterConfigSharedPtr + setupConfig(envoy::config::rbac::v3::RBAC::Action action) { envoy::extensions::filters::http::rbac::v3::RBAC config; envoy::config::rbac::v3::Policy policy; @@ -36,7 +39,7 @@ class RoleBasedAccessControlFilterTest : public testing::Test { policy_rules->add_rules()->set_destination_port(123); policy_rules->add_rules()->mutable_url_path()->mutable_path()->set_suffix("suffix"); policy.add_principals()->set_any(true); - config.mutable_rules()->set_action(envoy::config::rbac::v3::RBAC::ALLOW); + config.mutable_rules()->set_action(action); (*config.mutable_rules()->mutable_policies())["foo"] = policy; envoy::config::rbac::v3::Policy shadow_policy; @@ -44,13 +47,14 @@ class RoleBasedAccessControlFilterTest : public testing::Test { shadow_policy_rules->add_rules()->mutable_requested_server_name()->set_exact("xyz.cncf.io"); shadow_policy_rules->add_rules()->set_destination_port(456); shadow_policy.add_principals()->set_any(true); - config.mutable_shadow_rules()->set_action(envoy::config::rbac::v3::RBAC::ALLOW); + config.mutable_shadow_rules()->set_action(action); (*config.mutable_shadow_rules()->mutable_policies())["bar"] = shadow_policy; return std::make_shared(config, "test", store_); } - RoleBasedAccessControlFilterTest() : config_(setupConfig()), filter_(config_) {} + RoleBasedAccessControlFilterTest() + : config_(setupConfig(envoy::config::rbac::v3::RBAC::ALLOW)), filter_(config_) {} void SetUp() override { EXPECT_CALL(callbacks_, connection()).WillRepeatedly(Return(&connection_)); @@ -68,6 +72,21 @@ class RoleBasedAccessControlFilterTest : public testing::Test { ON_CALL(connection_, requestedServerName()).WillByDefault(Return(requested_server_name_)); } + void checkAccessLogMetadata(LogResult expected) { + if (expected != LogResult::Undecided) { + auto filter_meta = req_info_.dynamicMetadata().filter_metadata().at( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace); + EXPECT_EQ(expected == LogResult::Yes, + filter_meta.fields() + .at(Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().AccessLogKey) + .bool_value()); + } else { + EXPECT_EQ(req_info_.dynamicMetadata().filter_metadata().end(), + req_info_.dynamicMetadata().filter_metadata().find( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace)); + } + } + void setMetadata() { ON_CALL(req_info_, setDynamicMetadata(HttpFilterNames::get().Rbac, _)) .WillByDefault(Invoke([this](const std::string&, const ProtobufWkt::Struct& obj) { @@ -75,6 +94,15 @@ class RoleBasedAccessControlFilterTest : public testing::Test { Protobuf::MapPair(HttpFilterNames::get().Rbac, obj)); })); + + ON_CALL(req_info_, + setDynamicMetadata( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace, _)) + .WillByDefault(Invoke([this](const std::string&, const ProtobufWkt::Struct& obj) { + req_info_.metadata_.mutable_filter_metadata()->insert( + Protobuf::MapPair( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace, obj)); + })); } NiceMock callbacks_; @@ -82,8 +110,8 @@ class RoleBasedAccessControlFilterTest : public testing::Test { NiceMock req_info_; Stats::IsolatedStoreImpl store_; RoleBasedAccessControlFilterConfigSharedPtr config_; - RoleBasedAccessControlFilter filter_; + Network::Address::InstanceConstSharedPtr address_; std::string requested_server_name_; Http::TestRequestHeaderMapImpl headers_; @@ -92,6 +120,7 @@ class RoleBasedAccessControlFilterTest : public testing::Test { TEST_F(RoleBasedAccessControlFilterTest, Allowed) { setDestinationPort(123); + setMetadata(); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers_, false)); Http::MetadataMap metadata_map{{"metadata", "metadata"}}; @@ -102,11 +131,14 @@ TEST_F(RoleBasedAccessControlFilterTest, Allowed) { Buffer::OwnedImpl data(""); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(trailers_)); + + checkAccessLogMetadata(LogResult::Undecided); } TEST_F(RoleBasedAccessControlFilterTest, RequestedServerName) { setDestinationPort(999); setRequestedServerName("www.cncf.io"); + setMetadata(); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers_, false)); EXPECT_EQ(1U, config_->stats().allowed_.value()); @@ -117,10 +149,13 @@ TEST_F(RoleBasedAccessControlFilterTest, RequestedServerName) { Buffer::OwnedImpl data(""); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(trailers_)); + + checkAccessLogMetadata(LogResult::Undecided); } TEST_F(RoleBasedAccessControlFilterTest, Path) { setDestinationPort(999); + setMetadata(); auto headers = Http::TestRequestHeaderMapImpl{ {":method", "GET"}, @@ -129,6 +164,7 @@ TEST_F(RoleBasedAccessControlFilterTest, Path) { {":authority", "host"}, }; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers, false)); + checkAccessLogMetadata(LogResult::Undecided); } TEST_F(RoleBasedAccessControlFilterTest, Denied) { @@ -151,23 +187,65 @@ TEST_F(RoleBasedAccessControlFilterTest, Denied) { EXPECT_EQ("allowed", filter_meta.fields().at("shadow_engine_result").string_value()); EXPECT_EQ("bar", filter_meta.fields().at("shadow_effective_policy_id").string_value()); EXPECT_EQ("rbac_access_denied", callbacks_.details_); + checkAccessLogMetadata(LogResult::Undecided); } TEST_F(RoleBasedAccessControlFilterTest, RouteLocalOverride) { setDestinationPort(456); + setMetadata(); envoy::extensions::filters::http::rbac::v3::RBACPerRoute route_config; route_config.mutable_rbac()->mutable_rules()->set_action(envoy::config::rbac::v3::RBAC::DENY); NiceMock engine{route_config.rbac().rules()}; NiceMock per_route_config_{route_config}; - EXPECT_CALL(engine, allowed(_, _, _, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(engine, handleAction(_, _, _, _)).WillRepeatedly(Return(true)); EXPECT_CALL(per_route_config_, engine()).WillRepeatedly(ReturnRef(engine)); EXPECT_CALL(callbacks_.route_->route_entry_, perFilterConfig(HttpFilterNames::get().Rbac)) .WillRepeatedly(Return(&per_route_config_)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers_, true)); + checkAccessLogMetadata(LogResult::Undecided); +} + +// Log Tests +TEST_F(RoleBasedAccessControlFilterTest, ShouldLog) { + config_ = setupConfig(envoy::config::rbac::v3::RBAC::LOG); + filter_ = RoleBasedAccessControlFilter(config_); + filter_.setDecoderFilterCallbacks(callbacks_); + + setDestinationPort(123); + setMetadata(); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers_, false)); + EXPECT_EQ(1U, config_->stats().allowed_.value()); + EXPECT_EQ(0U, config_->stats().shadow_denied_.value()); + + Buffer::OwnedImpl data(""); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(trailers_)); + + checkAccessLogMetadata(LogResult::Yes); +} + +TEST_F(RoleBasedAccessControlFilterTest, ShouldNotLog) { + config_ = setupConfig(envoy::config::rbac::v3::RBAC::LOG); + filter_ = RoleBasedAccessControlFilter(config_); + filter_.setDecoderFilterCallbacks(callbacks_); + + setDestinationPort(456); + setMetadata(); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers_, false)); + EXPECT_EQ(1U, config_->stats().allowed_.value()); + EXPECT_EQ(0U, config_->stats().shadow_denied_.value()); + + Buffer::OwnedImpl data(""); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(trailers_)); + + checkAccessLogMetadata(LogResult::No); } } // namespace diff --git a/test/extensions/filters/network/rbac/filter_test.cc b/test/extensions/filters/network/rbac/filter_test.cc index 2e8fd2642da3..fe042854baa7 100644 --- a/test/extensions/filters/network/rbac/filter_test.cc +++ b/test/extensions/filters/network/rbac/filter_test.cc @@ -22,8 +22,10 @@ namespace RBACFilter { class RoleBasedAccessControlNetworkFilterTest : public testing::Test { public: - RoleBasedAccessControlFilterConfigSharedPtr setupConfig(bool with_policy = true, - bool continuous = false) { + RoleBasedAccessControlFilterConfigSharedPtr + setupConfig(bool with_policy = true, bool continuous = false, + envoy::config::rbac::v3::RBAC::Action action = envoy::config::rbac::v3::RBAC::ALLOW) { + envoy::extensions::filters::network::rbac::v3::RBAC config; config.set_stat_prefix("tcp."); @@ -34,7 +36,7 @@ class RoleBasedAccessControlNetworkFilterTest : public testing::Test { ".*cncf.io"); policy_rules->add_rules()->set_destination_port(123); policy.add_principals()->set_any(true); - config.mutable_rules()->set_action(envoy::config::rbac::v3::RBAC::ALLOW); + config.mutable_rules()->set_action(action); (*config.mutable_rules()->mutable_policies())["foo"] = policy; envoy::config::rbac::v3::Policy shadow_policy; @@ -42,7 +44,7 @@ class RoleBasedAccessControlNetworkFilterTest : public testing::Test { shadow_policy_rules->add_rules()->mutable_requested_server_name()->set_exact("xyz.cncf.io"); shadow_policy_rules->add_rules()->set_destination_port(456); shadow_policy.add_principals()->set_any(true); - config.mutable_shadow_rules()->set_action(envoy::config::rbac::v3::RBAC::ALLOW); + config.mutable_shadow_rules()->set_action(action); (*config.mutable_shadow_rules()->mutable_policies())["bar"] = shadow_policy; } @@ -72,6 +74,15 @@ class RoleBasedAccessControlNetworkFilterTest : public testing::Test { .WillByDefault(Return(requested_server_name_)); } + void checkAccessLogMetadata(bool expected) { + auto filter_meta = stream_info_.dynamicMetadata().filter_metadata().at( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace); + EXPECT_EQ(expected, + filter_meta.fields() + .at(Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().AccessLogKey) + .bool_value()); + } + void setMetadata() { ON_CALL(stream_info_, setDynamicMetadata(NetworkFilterNames::get().Rbac, _)) .WillByDefault(Invoke([this](const std::string&, const ProtobufWkt::Struct& obj) { @@ -79,6 +90,15 @@ class RoleBasedAccessControlNetworkFilterTest : public testing::Test { Protobuf::MapPair(NetworkFilterNames::get().Rbac, obj)); })); + + ON_CALL(stream_info_, + setDynamicMetadata( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace, _)) + .WillByDefault(Invoke([this](const std::string&, const ProtobufWkt::Struct& obj) { + stream_info_.metadata_.mutable_filter_metadata()->insert( + Protobuf::MapPair( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace, obj)); + })); } NiceMock callbacks_; @@ -173,6 +193,49 @@ TEST_F(RoleBasedAccessControlNetworkFilterTest, Denied) { EXPECT_EQ("allowed", filter_meta.fields().at("shadow_engine_result").string_value()); } +// Log Tests +TEST_F(RoleBasedAccessControlNetworkFilterTest, ShouldLog) { + config_ = setupConfig(true, false, envoy::config::rbac::v3::RBAC::LOG); + filter_ = std::make_unique(config_); + filter_->initializeReadFilterCallbacks(callbacks_); + + setDestinationPort(123); + setMetadata(); + + EXPECT_EQ(Network::FilterStatus::Continue, filter_->onData(data_, false)); + EXPECT_EQ(1U, config_->stats().allowed_.value()); + EXPECT_EQ(0U, config_->stats().shadow_denied_.value()); + + checkAccessLogMetadata(true); +} + +TEST_F(RoleBasedAccessControlNetworkFilterTest, ShouldNotLog) { + config_ = setupConfig(true, false, envoy::config::rbac::v3::RBAC::LOG); + filter_ = std::make_unique(config_); + filter_->initializeReadFilterCallbacks(callbacks_); + + setDestinationPort(456); + setMetadata(); + + EXPECT_EQ(Network::FilterStatus::Continue, filter_->onData(data_, false)); + EXPECT_EQ(1U, config_->stats().allowed_.value()); + EXPECT_EQ(0U, config_->stats().shadow_denied_.value()); + + checkAccessLogMetadata(false); +} + +TEST_F(RoleBasedAccessControlNetworkFilterTest, AllowNoChangeLog) { + setDestinationPort(123); + setMetadata(); + + EXPECT_EQ(Network::FilterStatus::Continue, filter_->onData(data_, false)); + + // Check that Allow action does not set access log metadata + EXPECT_EQ(stream_info_.dynamicMetadata().filter_metadata().end(), + stream_info_.dynamicMetadata().filter_metadata().find( + Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().CommonNamespace)); +} + } // namespace RBACFilter } // namespace NetworkFilters } // namespace Extensions From 9ad7d4ce5923fb053506cb76d47f93049a509a4c Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 10 Aug 2020 21:52:30 -0700 Subject: [PATCH 29/67] tracing: add baggage methods to Tracing::Span (#12260) * WIP add some header info on baggage Signed-off-by: Matthew Grossman * messing w/ stuff Signed-off-by: Matthew Grossman * get a compiling solution Signed-off-by: Matthew Grossman * lint Signed-off-by: Matthew Grossman * format again Signed-off-by: Matthew Grossman * formatting Signed-off-by: Matthew Grossman * change the test slightly Signed-off-by: Matthew Grossman * s/std::string/string_view Signed-off-by: Matthew Grossman * add note about baggage and child/parent spans Signed-off-by: Matthew Grossman * add in test for OpenTracingDriverTest Signed-off-by: Matthew Grossman * add in test for OpenCensus Signed-off-by: Matthew Grossman * add in baggage test for xray Signed-off-by: Matthew Grossman * add in zipkin tests Signed-off-by: Matthew Grossman * add in http tracer impl test Signed-off-by: Matthew Grossman * fix test for nulltracer Signed-off-by: Matthew Grossman * Add additional TODO for zipkin impl and add in comments for tests Signed-off-by: Matthew Grossman * add in draft docs Signed-off-by: Matthew Grossman * fix doc link Signed-off-by: Matthew Grossman * Inline empty methods per mklein's nit Signed-off-by: Jake Kaufman Co-authored-by: Jake Kaufman --- .../arch_overview/observability/tracing.rst | 14 ++++++++++++++ include/envoy/tracing/http_tracer.h | 16 ++++++++++++++++ source/common/tracing/http_tracer_impl.h | 2 ++ .../tracers/common/ot/opentracing_driver_impl.cc | 8 ++++++++ .../tracers/common/ot/opentracing_driver_impl.h | 2 ++ .../tracers/opencensus/opencensus_tracer_impl.cc | 4 ++++ source/extensions/tracers/xray/tracer.h | 4 ++++ .../tracers/zipkin/zipkin_tracer_impl.cc | 4 ++++ .../tracers/zipkin/zipkin_tracer_impl.h | 4 ++++ test/common/tracing/http_tracer_impl_test.cc | 2 ++ .../common/ot/opentracing_driver_impl_test.cc | 14 ++++++++++++++ .../lightstep/lightstep_tracer_impl_test.cc | 11 +++++++++++ .../extensions/tracers/opencensus/tracer_test.cc | 4 ++++ test/extensions/tracers/xray/tracer_test.cc | 10 ++++++++++ .../tracers/zipkin/zipkin_tracer_impl_test.cc | 8 ++++++++ test/mocks/tracing/mocks.h | 2 ++ 16 files changed, 109 insertions(+) diff --git a/docs/root/intro/arch_overview/observability/tracing.rst b/docs/root/intro/arch_overview/observability/tracing.rst index 26f057468d76..958b003a5d9a 100644 --- a/docs/root/intro/arch_overview/observability/tracing.rst +++ b/docs/root/intro/arch_overview/observability/tracing.rst @@ -113,3 +113,17 @@ request ID :ref:`config_http_conn_man_headers_x-request-id` (LightStep) or the trace ID configuration (Zipkin and Datadog). See :ref:`v3 API reference ` for more information on how to setup tracing in Envoy. + +Baggage +----------------------------- +Baggage provides a mechanism for data to be available throughout the entirety of a trace. +While metadata such as tags are usually communicated to collectors out-of-band, baggage data is injected into the actual +request context and available to applications during the duration of the request. This enables metadata to transparently +travel from the beginning of the request throughout your entire mesh without relying on application-specific modifications for +propagation. See `OpenTracing's documentation `_ for more information about baggage. + +Tracing providers have varying level of support for getting and setting baggage: + +* Lightstep (and any OpenTracing-compliant tracer) can read/write baggage +* Zipkin support is not yet implemented +* X-Ray and OpenCensus don't support baggage diff --git a/include/envoy/tracing/http_tracer.h b/include/envoy/tracing/http_tracer.h index 63da639e84ee..22b024ac97e0 100644 --- a/include/envoy/tracing/http_tracer.h +++ b/include/envoy/tracing/http_tracer.h @@ -158,6 +158,22 @@ class Span { * @param sampled whether the span and any subsequent child spans should be sampled */ virtual void setSampled(bool sampled) PURE; + + /** + * Retrieve a key's value from the span's baggage. + * This baggage data could've been set by this span or any parent spans. + * @param key baggage key + * @return the baggage's value for the given input key + */ + virtual std::string getBaggage(absl::string_view key) PURE; + + /** + * Set a key/value pair in the current span's baggage. + * All subsequent child spans will have access to this baggage. + * @param key baggage key + * @param key baggage value + */ + virtual void setBaggage(absl::string_view key, absl::string_view value) PURE; }; /** diff --git a/source/common/tracing/http_tracer_impl.h b/source/common/tracing/http_tracer_impl.h index 14cb47cbeb6a..760b4ed2bf3e 100644 --- a/source/common/tracing/http_tracer_impl.h +++ b/source/common/tracing/http_tracer_impl.h @@ -169,6 +169,8 @@ class NullSpan : public Span { void log(SystemTime, const std::string&) override {} void finishSpan() override {} void injectContext(Http::RequestHeaderMap&) override {} + void setBaggage(absl::string_view, absl::string_view) override {} + std::string getBaggage(absl::string_view) override { return std::string(); } SpanPtr spawnChild(const Config&, const std::string&, SystemTime) override { return SpanPtr{new NullSpan()}; } diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc index 080742af2ddd..cad01b83bb83 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc @@ -102,6 +102,14 @@ void OpenTracingSpan::log(SystemTime timestamp, const std::string& event) { finish_options_.log_records.emplace_back(std::move(record)); } +void OpenTracingSpan::setBaggage(absl::string_view key, absl::string_view value) { + span_->SetBaggageItem({key.data(), key.length()}, {value.data(), value.length()}); +} + +std::string OpenTracingSpan::getBaggage(absl::string_view key) { + return span_->BaggageItem({key.data(), key.length()}); +} + void OpenTracingSpan::injectContext(Http::RequestHeaderMap& request_headers) { if (driver_.propagationMode() == OpenTracingDriver::PropagationMode::SingleHeader) { // Inject the span context using Envoy's single-header format. diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.h b/source/extensions/tracers/common/ot/opentracing_driver_impl.h index d99ad7444dc5..2bfbddfe1886 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.h +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.h @@ -40,6 +40,8 @@ class OpenTracingSpan : public Tracing::Span, Logger::Loggable { */ void log(Envoy::SystemTime, const std::string&) override {} + // X-Ray doesn't support baggage, so noop these OpenTracing functions. + void setBaggage(absl::string_view, absl::string_view) override {} + std::string getBaggage(absl::string_view) override { return std::string(); } + /** * Creates a child span. * In X-Ray terms this creates a sub-segment and sets its parent ID to the current span's ID. diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc index a96d91aab565..8cf176d1fabc 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc @@ -35,6 +35,10 @@ void ZipkinSpan::log(SystemTime timestamp, const std::string& event) { span_.log(timestamp, event); } +// TODO(#11622): Implement baggage storage for zipkin spans +void ZipkinSpan::setBaggage(absl::string_view, absl::string_view) {} +std::string ZipkinSpan::getBaggage(absl::string_view) { return std::string(); } + void ZipkinSpan::injectContext(Http::RequestHeaderMap& request_headers) { // Set the trace-id and span-id headers properly, based on the newly-created span structure. request_headers.setReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h index 1624ddf59cc5..9cb39ea27e92 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h @@ -76,6 +76,10 @@ class ZipkinSpan : public Tracing::Span { void setSampled(bool sampled) override; + // TODO(#11622): Implement baggage storage for zipkin spans + void setBaggage(absl::string_view, absl::string_view) override; + std::string getBaggage(absl::string_view) override; + /** * @return a reference to the Zipkin::Span object. */ diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index 6bb18da079f8..ef1686bcc688 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -744,6 +744,8 @@ TEST(HttpNullTracerTest, BasicFunctionality) { span_ptr->setOperation("foo"); span_ptr->setTag("foo", "bar"); + span_ptr->setBaggage("key", "value"); + ASSERT_EQ("", span_ptr->getBaggage("baggage_key")); span_ptr->injectContext(request_headers); EXPECT_NE(nullptr, span_ptr->spawnChild(config, "foo", SystemTime())); diff --git a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc index 102bc6a2086c..011030dff5a4 100644 --- a/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc +++ b/test/extensions/tracers/common/ot/opentracing_driver_impl_test.cc @@ -99,6 +99,20 @@ TEST_F(OpenTracingDriverTest, FlushSpanWithLog) { EXPECT_EQ(expected_logs, driver_->recorder().top().logs); } +TEST_F(OpenTracingDriverTest, FlushSpanWithBaggage) { + setupValidDriver(); + + Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, + start_time_, {Tracing::Reason::Sampling, true}); + first_span->setBaggage("abc", "123"); + first_span->finishSpan(); + + const std::map expected_baggage = {{"abc", "123"}}; + + EXPECT_EQ(1, driver_->recorder().spans().size()); + EXPECT_EQ(expected_baggage, driver_->recorder().top().span_context.baggage); +} + TEST_F(OpenTracingDriverTest, TagSamplingFalseByDecision) { setupValidDriver(OpenTracingDriver::PropagationMode::TracerNative, {}); diff --git a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc index caba59fce679..ef657d6d54f5 100644 --- a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc +++ b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc @@ -718,6 +718,17 @@ TEST_F(LightStepDriverTest, SpawnChild) { EXPECT_FALSE(base2_context.empty()); } +TEST_F(LightStepDriverTest, GetAndSetBaggage) { + setupValidDriver(); + Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, + start_time_, {Tracing::Reason::Sampling, true}); + + std::string key = "key1"; + std::string value = "value1"; + span->setBaggage(key, value); + EXPECT_EQ(span->getBaggage(key), value); +} + } // namespace } // namespace Lightstep } // namespace Tracers diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index 6bc91183341a..88ed7f2f5983 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -123,6 +123,10 @@ TEST(OpenCensusTracerTest, Span) { child->finishSpan(); span->setSampled(false); // Abandon tracer. span->finishSpan(); + + // Baggage methods are a noop in opencensus and won't affect events. + span->setBaggage("baggage_key", "baggage_value"); + ASSERT_EQ("", span->getBaggage("baggage_key")); } // Retrieve SpanData from the OpenCensus trace exporter. diff --git a/test/extensions/tracers/xray/tracer_test.cc b/test/extensions/tracers/xray/tracer_test.cc index c82e73056a5f..caeb153def47 100644 --- a/test/extensions/tracers/xray/tracer_test.cc +++ b/test/extensions/tracers/xray/tracer_test.cc @@ -100,6 +100,16 @@ TEST_F(XRayTracerTest, NonSampledSpansNotSerialized) { span->finishSpan(); } +TEST_F(XRayTracerTest, BaggageNotImplemented) { + Tracer tracer{"" /*span name*/, std::move(broker_), server_.timeSource()}; + auto span = tracer.createNonSampledSpan(); + span->setBaggage("baggage_key", "baggage_value"); + span->finishSpan(); + + // Baggage isn't supported so getBaggage should always return empty + ASSERT_EQ("", span->getBaggage("baggage_key")); +} + TEST_F(XRayTracerTest, ChildSpanHasParentInfo) { NiceMock config; constexpr auto expected_span_name = "Service 1"; diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index bd51f1493e5c..0d1488e63bff 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -689,6 +689,14 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { EXPECT_FALSE(zipkin_zipkin_span4.annotations().empty()); EXPECT_EQ(timestamp_count, zipkin_zipkin_span4.annotations().back().timestamp()); EXPECT_EQ("abc", zipkin_zipkin_span4.annotations().back().value()); + + // ==== + // Test baggage noop + // ==== + Tracing::SpanPtr span5 = driver_->startSpan(config_, request_headers_, operation_name_, + start_time_, {Tracing::Reason::Sampling, true}); + span5->setBaggage("baggage_key", "baggage_value"); + EXPECT_EQ("", span5->getBaggage("baggage_key")); } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersTest) { diff --git a/test/mocks/tracing/mocks.h b/test/mocks/tracing/mocks.h index 8531027c7543..98a7a96ac513 100644 --- a/test/mocks/tracing/mocks.h +++ b/test/mocks/tracing/mocks.h @@ -37,6 +37,8 @@ class MockSpan : public Span { MOCK_METHOD(void, finishSpan, ()); MOCK_METHOD(void, injectContext, (Http::RequestHeaderMap & request_headers)); MOCK_METHOD(void, setSampled, (const bool sampled)); + MOCK_METHOD(void, setBaggage, (absl::string_view key, absl::string_view value)); + MOCK_METHOD(std::string, getBaggage, (absl::string_view key)); SpanPtr spawnChild(const Config& config, const std::string& name, SystemTime start_time) override { From 887637c1277134df9bea3ad9ac958e38ca69f6db Mon Sep 17 00:00:00 2001 From: DongRyeol Cha Date: Tue, 11 Aug 2020 16:16:59 +0900 Subject: [PATCH 30/67] build: Fix that custom docker image build (#12564) * build: Fix that custom docker image build The do_ci.sh builds the envoy and copies binary to under build-* directories. But recently, the arm64 build system was added but it does not consider the previous custom docker image build. It is very useful to build the custom docker image and test it within k8s environment. So, this patch support previous behavior again so that we can build the custom docker image. If some more cpu architectures are added, we can simple extends more cpu architectures. Signed-off-by: DongRyeol Cha --- .gitignore | 2 +- ci/build_setup.sh | 4 +++- ci/do_ci.sh | 25 ++++++++++++++++++------- ci/run_envoy_docker.sh | 2 +- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 7b7c6ff04d58..134967bc2bb7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ /bazel-* BROWSE /build -/build_* *.bzlc .cache .clangd @@ -34,3 +33,4 @@ clang.bazelrc user.bazelrc CMakeLists.txt cmake-build-debug +/linux diff --git a/ci/build_setup.sh b/ci/build_setup.sh index aa21bbadb232..93330224137d 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -9,8 +9,10 @@ export PPROF_PATH=/thirdparty_build/bin/pprof [ -z "${NUM_CPUS}" ] && NUM_CPUS=`grep -c ^processor /proc/cpuinfo` [ -z "${ENVOY_SRCDIR}" ] && export ENVOY_SRCDIR=/source [ -z "${ENVOY_BUILD_TARGET}" ] && export ENVOY_BUILD_TARGET=//source/exe:envoy-static +[ -z "${ENVOY_BUILD_ARCH}" ] && export ENVOY_BUILD_ARCH=$(uname -m) echo "ENVOY_SRCDIR=${ENVOY_SRCDIR}" echo "ENVOY_BUILD_TARGET=${ENVOY_BUILD_TARGET}" +echo "ENVOY_BUILD_ARCH=${ENVOY_BUILD_ARCH}" function setup_gcc_toolchain() { if [[ ! -z "${ENVOY_STDLIB}" && "${ENVOY_STDLIB}" != "libstdc++" ]]; then @@ -83,7 +85,7 @@ export BAZEL_BUILD_OPTIONS=" ${BAZEL_OPTIONS} --verbose_failures --show_task_fin --test_output=errors --repository_cache=${BUILD_DIR}/repository_cache --experimental_repository_cache_hardlinks \ ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}" -[[ "$(uname -m)" == "aarch64" ]] && BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=HEAPCHECK=" +[[ "${ENVOY_BUILD_ARCH}" == "aarch64" ]] && BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=HEAPCHECK=" [[ "${BAZEL_EXPUNGE}" == "1" ]] && bazel clean --expunge diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 3218361ce81d..40fa5312b805 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -16,7 +16,17 @@ SRCDIR="${PWD}" . "$(dirname "$0")"/build_setup.sh $build_setup_args cd "${SRCDIR}" +if [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]]; then + BUILD_ARCH_DIR="/linux/amd64" +elif [[ "${ENVOY_BUILD_ARCH}" == "aarch64" ]]; then + BUILD_ARCH_DIR="/linux/arm64" +else + # Fall back to use the ENVOY_BUILD_ARCH itself. + BUILD_ARCH_DIR="/linux/${ENVOY_BUILD_ARCH}" +fi + echo "building using ${NUM_CPUS} CPUs" +echo "building for ${ENVOY_BUILD_ARCH}" function collect_build_profile() { declare -g build_profile_count=${build_profile_count:-1} @@ -52,18 +62,19 @@ function cp_binary_for_outside_access() { function cp_binary_for_image_build() { # TODO(mattklein123): Replace this with caching and a different job which creates images. + local BASE_TARGET_DIR="${ENVOY_SRCDIR}${BUILD_ARCH_DIR}" echo "Copying binary for image build..." - mkdir -p "${ENVOY_SRCDIR}"/build_"$1" - cp -f "${ENVOY_DELIVERY_DIR}"/envoy "${ENVOY_SRCDIR}"/build_"$1" - mkdir -p "${ENVOY_SRCDIR}"/build_"$1"_stripped - strip "${ENVOY_DELIVERY_DIR}"/envoy -o "${ENVOY_SRCDIR}"/build_"$1"_stripped/envoy + mkdir -p "${BASE_TARGET_DIR}"/build_"$1" + cp -f "${ENVOY_DELIVERY_DIR}"/envoy "${BASE_TARGET_DIR}"/build_"$1" + mkdir -p "${BASE_TARGET_DIR}"/build_"$1"_stripped + strip "${ENVOY_DELIVERY_DIR}"/envoy -o "${BASE_TARGET_DIR}"/build_"$1"_stripped/envoy # Copy for azp which doesn't preserve permissions, creating a tar archive - tar czf "${ENVOY_BUILD_DIR}"/envoy_binary.tar.gz -C "${ENVOY_SRCDIR}" build_"$1" build_"$1"_stripped + tar czf "${ENVOY_BUILD_DIR}"/envoy_binary.tar.gz -C "${BASE_TARGET_DIR}" build_"$1" build_"$1"_stripped # Remove binaries to save space, only if BUILD_REASON exists (running in AZP) [[ -z "${BUILD_REASON}" ]] || \ - rm -rf "${ENVOY_SRCDIR}"/build_"$1" "${ENVOY_SRCDIR}"/build_"$1"_stripped "${ENVOY_DELIVERY_DIR}"/envoy \ + rm -rf "${BASE_TARGET_DIR}"/build_"$1" "${BASE_TARGET_DIR}"/build_"$1"_stripped "${ENVOY_DELIVERY_DIR}"/envoy \ bazel-bin/"${ENVOY_BIN}" } @@ -117,7 +128,7 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then # toolchain is kept consistent. This ifdef is checked in # test/common/stats/stat_test_utility.cc when computing # Stats::TestUtil::MemoryTest::mode(). - [[ "$(uname -m)" == "x86_64" ]] && BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=ENVOY_MEMORY_TEST_EXACT=true" + [[ "${ENVOY_BUILD_ARCH}" == "x86_64" ]] && BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=ENVOY_MEMORY_TEST_EXACT=true" setup_clang_toolchain echo "Testing ${TEST_TARGETS}" diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 886a2347d378..ca29667c14c9 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -28,7 +28,7 @@ docker run --rm ${ENVOY_DOCKER_OPTIONS} -e HTTP_PROXY=${http_proxy} -e HTTPS_PRO -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE -e ENVOY_STDLIB -e BUILD_REASON \ -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE -e FUZZIT_API_KEY -e ENVOY_BUILD_IMAGE \ -e ENVOY_SRCDIR -e ENVOY_BUILD_TARGET -e SYSTEM_PULLREQUEST_TARGETBRANCH -e SYSTEM_PULLREQUEST_PULLREQUESTNUMBER \ - -e GCS_ARTIFACT_BUCKET -e BUILD_SOURCEBRANCHNAME -e BAZELISK_BASE_URL \ + -e GCS_ARTIFACT_BUCKET -e BUILD_SOURCEBRANCHNAME -e BAZELISK_BASE_URL -e ENVOY_BUILD_ARCH \ -v "$PWD":/source --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${ENVOY_BUILD_IMAGE}" \ /bin/bash -lc "groupadd --gid $(id -g) -f envoygroup && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home \ --home-dir /build envoybuild && usermod -a -G pcap envoybuild && sudo -EHs -u envoybuild bash -c \"cd /source && $*\"" From 276c978576b252f0f20d8531fe9dd90d966ede42 Mon Sep 17 00:00:00 2001 From: Sotiris Nanopoulos Date: Tue, 11 Aug 2020 05:21:24 -0700 Subject: [PATCH 31/67] Fixes grpc tests on Windows (#12433) 1. Adds localhost to the upstreamcert.cfg 2. Makes fake_upstream TcpListenerSocket connect to the localhost instead of any address. 3. GRPC tests are no longer failing on windows. Risk Level: Low Testing: N/A Signed-off-by: Sotiris Nanopoulos --- test/common/grpc/BUILD | 2 - test/config/integration/certs/cacert.pem | 42 ++++++++-------- test/config/integration/certs/cakey.pem | 50 +++++++++---------- .../integration/certs/client_ecdsacert.pem | 40 +++++++-------- .../integration/certs/client_ecdsacert_hash.h | 4 +- .../integration/certs/client_ecdsakey.pem | 6 +-- test/config/integration/certs/clientcert.pem | 48 +++++++++--------- .../integration/certs/clientcert_hash.h | 4 +- test/config/integration/certs/clientkey.pem | 50 +++++++++---------- .../integration/certs/server_ecdsacert.pem | 40 +++++++-------- .../integration/certs/server_ecdsacert_hash.h | 4 +- .../integration/certs/server_ecdsakey.pem | 6 +-- test/config/integration/certs/servercert.pem | 48 +++++++++--------- .../integration/certs/servercert_hash.h | 4 +- test/config/integration/certs/serverkey.pem | 50 +++++++++---------- .../integration/certs/upstreamcacert.pem | 36 ++++++------- .../integration/certs/upstreamcakey.pem | 50 +++++++++---------- .../config/integration/certs/upstreamcert.cfg | 4 +- .../config/integration/certs/upstreamcert.pem | 36 ++++++------- .../integration/certs/upstreamcert_hash.h | 4 +- test/config/integration/certs/upstreamkey.pem | 50 +++++++++---------- .../certs/upstreamlocalhostcert.pem | 36 ++++++------- .../certs/upstreamlocalhostcert_hash.h | 4 +- .../certs/upstreamlocalhostkey.pem | 50 +++++++++---------- .../extensions/grpc_credentials/aws_iam/BUILD | 1 - .../file_based_metadata/BUILD | 1 - test/integration/fake_upstream.cc | 4 +- 27 files changed, 335 insertions(+), 339 deletions(-) diff --git a/test/common/grpc/BUILD b/test/common/grpc/BUILD index edfdfbda404a..46b7dd318888 100644 --- a/test/common/grpc/BUILD +++ b/test/common/grpc/BUILD @@ -163,8 +163,6 @@ envoy_cc_test_library( envoy_cc_test( name = "grpc_client_integration_test", srcs = ["grpc_client_integration_test.cc"], - # Fails on windows; Connection is terminated early testing BasicStream/IPv4_EnvoyGrpc - tags = ["fails_on_windows"], deps = [ ":grpc_client_integration_test_harness_lib", "//source/common/grpc:async_client_lib", diff --git a/test/config/integration/certs/cacert.pem b/test/config/integration/certs/cacert.pem index 021f465ac3ef..c54bd51a3074 100644 --- a/test/config/integration/certs/cacert.pem +++ b/test/config/integration/certs/cacert.pem @@ -1,23 +1,23 @@ -----BEGIN CERTIFICATE----- -MIID0jCCArqgAwIBAgIJAJxgLCQiz7YlMA0GCSqGSIb3DQEBCwUAMHYxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2luZWVyaW5nMRAw -DgYDVQQDDAdUZXN0IENBMB4XDTE4MTIxNzIwMTgwMFoXDTIwMTIxNjIwMTgwMFow -djELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcMDVNh -biBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5naW5l -ZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw -ggEKAoIBAQCpZHOUq+nidd+Gz44RC80QG9De9jcFUStEMGucXlnvvp2cH3GV4GmO -IZPdCwasxuruO3VM/Yt8tUAO2OrTQayuL9GXTt8MTpkCrviebMBzjYjbgyLgDpZy -cMoEJjBx0JsfQV+9IUDROLlIehTYzjcIWuLEOqMjZXQQCOI+jA3CEWZx1TFhRWhi -9aBnQQzWCSZPV5ErKSSRg2T2Xnuhusue7ETtgSt36hDrOxLhJaeS1/YlovyhX94j -JPhASK3LutJUDO2tk8L713Y3WHkFzDMfkGrklRbBB/ZKXRRGiJDWElpbUCUVFbuw -7laBtTn0t74DQxBXqal9sIr9vV7LLQszAgMBAAGjYzBhMA8GA1UdEwEB/wQFMAMB -Af8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBQUM9b2kmz7njy/vuxkzKiwDLZN -5DAfBgNVHSMEGDAWgBQUM9b2kmz7njy/vuxkzKiwDLZN5DANBgkqhkiG9w0BAQsF -AAOCAQEAkWqORue+2exZldWbYaDUX3ANP0ATBNIbZM70uTNO8Iy+r5Fvxtae/PsV -Iac9LzVY5dY5eqIND9wo7osFfxEhJdJn+/tpTU2h9IhsuWMm0Ogj87NS3sy0xwDc -xBhnVXI8nCDYU3qU3p+AeC0VfEbNb+dRKHH/FL77jvIL64GP/WGxxS9u7LRTUUoR -g97ZWeayKEsHAicRao4/k3jgpNIUN0BOlkjLvCe1ExU0id5R3UtdITmbuSSe6GJx -j8xsEV8PxmOIaJ/M+fqE+Zi2Ljp3a+9X/nLakR6ohMNTbrGMQWrGIpFqCj6pIwek -6Uemmmca+JeVohl8P3enMlW1d6/24w== +MIID3TCCAsWgAwIBAgIUdCu/mLip3X/We37vh3BA9u/nxakwDQYJKoZIhvcNAQEL +BQAwdjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwHhcNMjAwODA1MTkxNjAwWhcNMjIw +ODA1MTkxNjAwWjB2MQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEW +MBQGA1UEBwwNU2FuIEZyYW5jaXNjbzENMAsGA1UECgwETHlmdDEZMBcGA1UECwwQ +THlmdCBFbmdpbmVlcmluZzEQMA4GA1UEAwwHVGVzdCBDQTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBALu2Ihi4DmaQG7zySZlWyM9SjxOXCI5840V7Hn0C +XoiI8sQQmKSC2YCzsaphQoJ0lXCi6Y47o5FkooYyLeNDQTGS0nh+IWm5RCyochtO +fnaKPv/hYxhpyFQEwkJkbF1Zt1s6j2rq5MzmbWZx090uXZEE82DNZ9QJaMPu6VWt +iwGoGoS5HF5HNlUVxLNUsklNH0ZfDafR7/LC2ty1vO1c6EJ6yCGiyJZZ7Ilbz27Q +HPAUd8CcDNKCHZDoMWkLSLN3Nj1MvPVZ5HDsHiNHXthP+zV8FQtloAuZ8Srsmlyg +rJREkc7gF3f6HrH5ShNhsRFFc53NUjDbYZuha1u4hiOE8lcCAwEAAaNjMGEwDwYD +VR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJZL2ixTtL6V +xpNz4qekny4NchiHMB8GA1UdIwQYMBaAFJZL2ixTtL6VxpNz4qekny4NchiHMA0G +CSqGSIb3DQEBCwUAA4IBAQAcgG+AaCdrUFEVJDn9UsO7zqzQ3c1VOp+WAtAU8OQK +Oc4vJYVVKpDs8OZFxmukCeqm1gz2zDeH7TfgCs5UnLtkplx1YO1bd9qvserJVHiD +LAK+Yl24ZEbrHPaq0zI1RLchqYUOGWmi51pcXi1gsfc8DQ3GqIXoai6kYJeV3jFJ +jxpQSR32nx6oNN/6kVKlgmBjlWrOy7JyDXGim6Z97TzmS6Clctewmw/5gZ9g+M8e +g0ZdFbFkNUjzSNm44hiDX8nR6yJRn+gLaARaJvp1dnT+MlvofZuER17WYKH4OyMs +ie3qKR3an4KC20CtFbpZfv540BVuTTOCtQ5xqZ/LTE78 -----END CERTIFICATE----- diff --git a/test/config/integration/certs/cakey.pem b/test/config/integration/certs/cakey.pem index 030b80b3d860..c545e1d17866 100644 --- a/test/config/integration/certs/cakey.pem +++ b/test/config/integration/certs/cakey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAqWRzlKvp4nXfhs+OEQvNEBvQ3vY3BVErRDBrnF5Z776dnB9x -leBpjiGT3QsGrMbq7jt1TP2LfLVADtjq00Gsri/Rl07fDE6ZAq74nmzAc42I24Mi -4A6WcnDKBCYwcdCbH0FfvSFA0Ti5SHoU2M43CFrixDqjI2V0EAjiPowNwhFmcdUx -YUVoYvWgZ0EM1gkmT1eRKykkkYNk9l57obrLnuxE7YErd+oQ6zsS4SWnktf2JaL8 -oV/eIyT4QEity7rSVAztrZPC+9d2N1h5BcwzH5Bq5JUWwQf2Sl0URoiQ1hJaW1Al -FRW7sO5WgbU59Le+A0MQV6mpfbCK/b1eyy0LMwIDAQABAoIBAHFGvYwkUqmgTbRn -RAfeLmmhUFJpsG2b1CUrhCrzZY1PmTJ4TIr/oVbs2WauIu6TrzNVC6JKw2bIBmhn -YtGXT5TEYZKfqcUfIm+K9rNq4l/jvCufTEktOCqbhlyz9R2HdNS38QAXJrNDDZSM -HzjE3kR2EsNKuyHGjJDUgAd3vROTXLuNxhfO+he84NTz6hPlzNOGRJDOnacIp1T1 -qbQdlHhqxJBVyWyjAYR38maBrleLZqV27Fd0sBzuUkU7i6vHRFHp4pD4PAzkzAsS -DMqCmVF2cM83BT9qz8TcP4wd4+hj9OG2QbIe1zfhUfYS8v/bNqwtIV4WlbyW3P65 -ynXSaAECgYEA0XptQYYrPeSLkEQNLPfDu4BnTE4X65exl8c5+qMWg3aECbdaxWqi -VmjNlDyzz0w3zzSRShNR7/6fSWULrWwdSCRbpqiU1+xSUrPXCT+tqI9CVqVl452E -rGHiPZgy7ljb1x/mfrhA8fASrp57Xze4DLqYEUI5fiYBcJRhVqGZvTMCgYEAzwMA -+wbe2qyi9CuiEfDY3sv5+gfLkYUh3yR7dwrdqccAnOgG0nOv4LDwaoW4Fk2aAYdw -GvutdUntXtVc77Ha613cXCL38w58r890EpQuDoSTvZlo6K7lrB0ZFpUdefvpP5eL -4hrkRXes4TeshjLT8C+0Fo7+2XbZfWx2ZHrzegECgYEAh3rYwrIVsXfo06tPoi+0 -RcZsCKvRSKvZTkKpuvJTkz7JcsdFS70FtUEfBKql2IKA7eAfv3rzWXaiaoORo93y -qj/pjsYlTekn7RknEHJAzG2rCAL8/NNZhWvhONkAx6pstJuLJZXhWxhb3NffDtwo -iwL7at4b9Px7neY5diAaIIUCgYA/OXujL4YA45khWfI16IlUAphmdNsHptGhhVLw -GLF6mPzm7zamMA8XYPMMlaqTpT/UF7l1hEiF+f41aJTp4Dgsio4y1btE0LfkOkgJ -JJisdnFpBuGzrzcWSgzPiNtn1jh246IlfHEbhmGWp5pZokx4nxkxiprrcBEc7XN7 -XNHgAQKBgQCgiC8H2bpqmLii5xp0ZyyzUYn6sOar0nmKNcwpoPPIaOXNyIk7Ildn -3bocdUVFL/oYJFav2cmfeLOaetSEqYzMZO2OzF+OHT3wLBIhUFKgHIuStHUScCAR -FHsqaZxQEdI187t9xlVPGRYTMnKYdvM8pZXe3VFMrNOwRM01xoqSvw== +MIIEpgIBAAKCAQEAu7YiGLgOZpAbvPJJmVbIz1KPE5cIjnzjRXsefQJeiIjyxBCY +pILZgLOxqmFCgnSVcKLpjjujkWSihjIt40NBMZLSeH4hablELKhyG05+doo+/+Fj +GGnIVATCQmRsXVm3WzqPaurkzOZtZnHT3S5dkQTzYM1n1Alow+7pVa2LAagahLkc +Xkc2VRXEs1SySU0fRl8Np9Hv8sLa3LW87VzoQnrIIaLIllnsiVvPbtAc8BR3wJwM +0oIdkOgxaQtIs3c2PUy89VnkcOweI0de2E/7NXwVC2WgC5nxKuyaXKCslESRzuAX +d/oesflKE2GxEUVznc1SMNthm6FrW7iGI4TyVwIDAQABAoIBAQCasKW4qTV04B17 +wE9WxmYGNIskIbszcUf54lRlwKYW7oThfqvMJukHXw5y0mP1Dg55HEhMpmlNUBl/ +barTNoFrUQuRsJ/oeHzuMIKYbj9ZgOQaCquXWtV0J9fOzuNeqqinzcKS4bBcCyjs +27E0/Riugd3vUFbYLkjf7urraHC9k1mqMrTgTcQct3LO3oUG0cd2ANX6fMrE7Hfw +ZNPf9U+G0/QQIvRCn7xAzWNo+kwwrd/Yz4/S1YKDXeAkRHGlqjoXnrucwRYbBfsV +zg66mGvtlTrYjKgz8GfKZD8Azz9LVVXbN/2P6WAL/vTeNfMCV4VDGVk7BNs7UU7X +C1vetPJBAoGBAN8zNzdw/1tE6gmHzN3Hytk76t7uLA5XKlH81f7akRe/JqZ1urqi +OgJz1coLsJeRbIczSuF6qpsBVufwGUV8pQbF5JQPhS74MESlFffJ8knDr+6AHFbn +ow/8ZmDPyBBZIRsYdZZpqjqlqWjqnpYKMwcXmf5Yiq1G7FUdBEtrcObDAoGBANdL +1ihVvRWZsN8gYC9Wb+PxKNqaKTZZ/1BGPHmuMB28r4F6C8EtLQCReiuBPiLWs7FE +7hSeFTgLEncBK9nVRYpG3W6UKowBV16mr7tGJpoYVoUNUT7SxiwZWHx+VLkTfWex +iNYv/Ycxl6iYdkXzuHqUinxccEziKqEu6zMzd5TdAoGBAKEjRo/eIlzwCc7LndnX +rdjbaxt6849+2mzKjmwpu2pbdDnk8ORgzmSK4CO4AMvMD4AkRcE3YAf8FZPpQTVr +YXDcWcOS2OIqCB7m2E9GGoeqoU8callLbevSms7180fqMP5w0CPBMUaZ5w55o/hK +cMCEB4cawTOL6n8gLcONU7slAoGBALCT826be3yW1Bj8ncbVdumV5nL8U2bPg3Zc +VMdb1Pzev3dLGQ70NV+s8W1zD/pU64YtybLBQRf5BMjz/fooUGOr4XsLLKYth3IK +9kB7tbdW1MdFd+g1yPFsTEW2+1fcI1ODqX46WA6k3wUZHpAa56gp4jdDPZvhNyOB +rsgMozxFAoGBAJuaqAWXFA2wy+zDw8P6rhRDF52DmNMYeVBvFGtBOBy0BYavsesj +qRMzdDdhrQMzYqLCnvurEytaLscM+Jltzt1ImWtNwXoTg1O+Cl01wzBWK/NGrMBg +tiHVT2ojF3hIyFjr9Z7IgcosyvaNTHnBxIKUy2tiAprruYOwhH9kXdHX -----END RSA PRIVATE KEY----- diff --git a/test/config/integration/certs/client_ecdsacert.pem b/test/config/integration/certs/client_ecdsacert.pem index d32aea145091..f982784dea6a 100644 --- a/test/config/integration/certs/client_ecdsacert.pem +++ b/test/config/integration/certs/client_ecdsacert.pem @@ -1,22 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDmTCCAoGgAwIBAgIJAILStmLgUUcYMA0GCSqGSIb3DQEBCwUAMHYxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2luZWVyaW5nMRAw -DgYDVQQDDAdUZXN0IENBMB4XDTE4MTIxNzIwMTgwMFoXDTIwMTIxNjIwMTgwMFow -gagxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1T -YW4gRnJhbmNpc2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2lu -ZWVyaW5nMRswGQYDVQQDDBJUZXN0IEZyb250ZW5kIFRlYW0xJTAjBgkqhkiG9w0B -CQEWFmZyb250ZW5kLXRlYW1AbHlmdC5jb20wWTATBgcqhkjOPQIBBggqhkjOPQMB -BwNCAAT9UPlVN7p2GE0w7a7G7v+AqYYKwpsI1exZyLK4MiEBdiFxnQnjPP0Cfjpq -8PvQCZdthQ7+WL+HhRirpCHPaVHno4HBMIG+MAwGA1UdEwEB/wQCMAAwCwYDVR0P -BAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATBCBgNVHREEOzA5 -hh9zcGlmZmU6Ly9seWZ0LmNvbS9mcm9udGVuZC10ZWFtgghseWZ0LmNvbYIMd3d3 -Lmx5ZnQuY29tMB0GA1UdDgQWBBQLeMuyYBahRJ6gH43BRFYhMfBYxTAfBgNVHSME -GDAWgBQUM9b2kmz7njy/vuxkzKiwDLZN5DANBgkqhkiG9w0BAQsFAAOCAQEAEh3S -cW+nQzgbRCbfDkj+qcEslAOWJc6pNf6npKHtfn78YehVUKOiKkMdqTcW4zuhGjYA -ifVkoyTgTHyqdxkNdgELIi7/7JDY0N73AgndCpA/F/OqqMHATVtUMzTZzk7EB+yx -L6KyGuXVH6WtNnJ9Zo+f87FgjCtVZvnWqZKEynqtBRv930UaMkPPXQ/YLsrjkMC0 -VX2cyUG6tblYNxCeT73SRTs07KeT9wXOoZaUNxBJo0zPUYk3D1gEaCVCPMpqIOUF -kHkEPCdBDeGE1/UT/rPeFcWI+fCpKDU7c41ojMQ4/HxxorecyzBHxCpuqKZFGzD5 -+SAdcodf60sCDUt8lA== +MIIDpDCCAoygAwIBAgIUJuVBh0FKfFgIcO++ljWm7D47eYgwDQYJKoZIhvcNAQEL +BQAwdjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwHhcNMjAwODA1MTkxNjAyWhcNMjIw +ODA1MTkxNjAyWjCBqDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsM +EEx5ZnQgRW5naW5lZXJpbmcxGzAZBgNVBAMMElRlc3QgRnJvbnRlbmQgVGVhbTEl +MCMGCSqGSIb3DQEJARYWZnJvbnRlbmQtdGVhbUBseWZ0LmNvbTBZMBMGByqGSM49 +AgEGCCqGSM49AwEHA0IABFWdfntWW5wivQyk9j45hLFf7QjInjo4H8up56yUkCcm +n7ewQ9BEPoJ74r5ro/6nPBiRiTx1aolAjDPhgOfUZiWjgcEwgb4wDAYDVR0TAQH/ +BAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMB +MEIGA1UdEQQ7MDmGH3NwaWZmZTovL2x5ZnQuY29tL2Zyb250ZW5kLXRlYW2CCGx5 +ZnQuY29tggx3d3cubHlmdC5jb20wHQYDVR0OBBYEFAB5r3dICa3pCN0DVINNnm3K +DhmKMB8GA1UdIwQYMBaAFJZL2ixTtL6VxpNz4qekny4NchiHMA0GCSqGSIb3DQEB +CwUAA4IBAQBel+R+1BpxpPhUGwkSbZBTY4zqU8w8Zx9eCCbpeAi96Qylg++nl88H +ZInHLsC77wfzSf8vKWpsA9KDMB4R4njN1WLsQOKYWJmOE3K+AUuPmqP6mYyAju5k +gGSg6BIhb5+HtkmvvF56qyuc4GWH8Ab/BhPMRli7h4cBMuSBoZeFoxNbIJfJDJ/P +ZEkHqe1NsI9VG6pmGZ1adBMf04yE+muv+s43Wl/Ry/7W3Ae28XkxmS/c0wCq5h2u +DMn6nnjDIzJ7tUqfWfacr8QZCQDYQKdBrE+OPw6bgK2p++jXfQDD277FcnMG6je/ +ZbqoW4tPXfdl/INMbS1j2h59RTrXGZaR -----END CERTIFICATE----- diff --git a/test/config/integration/certs/client_ecdsacert_hash.h b/test/config/integration/certs/client_ecdsacert_hash.h index 787f4511b3b4..f765514c363d 100644 --- a/test/config/integration/certs/client_ecdsacert_hash.h +++ b/test/config/integration/certs/client_ecdsacert_hash.h @@ -1,3 +1,3 @@ // NOLINT(namespace-envoy) -constexpr char TEST_CLIENT_ECDSA_CERT_HASH[] = "DC:B2:C0:82:AD:8C:C3:4E:06:2E:65:15:F9:F7:39:D0:4C:" - "8A:60:AF:1E:B6:01:18:C5:90:AE:AE:47:7D:B8:FC"; +constexpr char TEST_CLIENT_ECDSA_CERT_HASH[] = "9F:F2:B4:5A:72:E4:79:82:F4:5C:B1:49:8A:EF:12:53:9C:" + "A7:AB:0A:61:DF:79:2F:D8:8D:4E:29:89:28:03:07"; diff --git a/test/config/integration/certs/client_ecdsakey.pem b/test/config/integration/certs/client_ecdsakey.pem index e352e35a540f..c41cf1b757cc 100644 --- a/test/config/integration/certs/client_ecdsakey.pem +++ b/test/config/integration/certs/client_ecdsakey.pem @@ -2,7 +2,7 @@ BggqhkjOPQMBBw== -----END EC PARAMETERS----- -----BEGIN EC PRIVATE KEY----- -MHcCAQEEIOoeoqRYQ6raBXh0ejEFGvOvP8XNgbKnj7tDYDLFH7t1oAoGCCqGSM49 -AwEHoUQDQgAE/VD5VTe6dhhNMO2uxu7/gKmGCsKbCNXsWciyuDIhAXYhcZ0J4zz9 -An46avD70AmXbYUO/li/h4UYq6Qhz2lR5w== +MHcCAQEEIJjBCnRPxmRg21Jdlt8MAQGqtD6ilFK2bsxx5twZklmzoAoGCCqGSM49 +AwEHoUQDQgAEVZ1+e1ZbnCK9DKT2PjmEsV/tCMieOjgfy6nnrJSQJyaft7BD0EQ+ +gnvivmuj/qc8GJGJPHVqiUCMM+GA59RmJQ== -----END EC PRIVATE KEY----- diff --git a/test/config/integration/certs/clientcert.pem b/test/config/integration/certs/clientcert.pem index edca455abfba..e56b5aa5fa1b 100644 --- a/test/config/integration/certs/clientcert.pem +++ b/test/config/integration/certs/clientcert.pem @@ -1,26 +1,26 @@ -----BEGIN CERTIFICATE----- -MIIEZDCCA0ygAwIBAgIJAILStmLgUUcXMA0GCSqGSIb3DQEBCwUAMHYxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2luZWVyaW5nMRAw -DgYDVQQDDAdUZXN0IENBMB4XDTE4MTIxNzIwMTgwMFoXDTIwMTIxNjIwMTgwMFow -gagxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1T -YW4gRnJhbmNpc2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2lu -ZWVyaW5nMRswGQYDVQQDDBJUZXN0IEZyb250ZW5kIFRlYW0xJTAjBgkqhkiG9w0B -CQEWFmZyb250ZW5kLXRlYW1AbHlmdC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IB -DwAwggEKAoIBAQCouLlGbMfpOV5Sh/zbUNo6GHzhpB1Q/UqxzkdmgdF4tzm3w83Q -qka2/Q4R4/Xi3gmbUlszfw2Ax8ouIbPlx5m/1/ytvUVOcyvksu+ae92WP62nukL3 -RLLt4ctvSiA0i9FeFrNmuM+R/cSdMf9tl20heL2/bgCmIlNr6i+KMsZQlWm3GRna -rfh8RWwyzqVDLLTvOO+h9jxHfKanN1BwIm0RYHRgxfyehel4av6t9xsylE4dtGvk -4DO7aO98IfNtn6rHyuIY2g8Wp4Mwbs9Z+uXSCc8/NTCOm9GSvG2hJSMIdqdnETLf -5eTiRUB3FF4el3nXNhOTFr0iAydL7bIYqkprAgMBAAGjgcEwgb4wDAYDVR0TAQH/ -BAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMB -MEIGA1UdEQQ7MDmGH3NwaWZmZTovL2x5ZnQuY29tL2Zyb250ZW5kLXRlYW2CCGx5 -ZnQuY29tggx3d3cubHlmdC5jb20wHQYDVR0OBBYEFJD5l1K7mp/dUUNXvXPBWJ5l -oJOnMB8GA1UdIwQYMBaAFBQz1vaSbPuePL++7GTMqLAMtk3kMA0GCSqGSIb3DQEB -CwUAA4IBAQCJOUQB5d8ZCEeMrY0jLwefY8L0UluhPFWNlw1t2LyDjAa8qRNkWJ/2 -bky7IHBMxPQIdYBVPsQGOQ4bkg3S7Eqyc0WZYpLlKEQeUFPG752642GInzjgY3KG -f1hIHz1quIYARjF5GJ+buZpw3DgcGDnhYygFQDWqgyRnfz84M1ycEx06yHidupyp -eMHZHXcrSXPcGin7a6tBEppDFm5CcrJQ2hySDVkl9qnbgHr0+0JZg/Qekik4aWv5 -ACWk3wTxIPUv6mc8kbBMRMPkETzWt4m/qdLnUUhFKJdyACPlb6onJe1TGuYJvc2x -rNV3U8Yo8a1iskQtHqNfc+kqVd3MpgsB +MIIEbzCCA1egAwIBAgIUJuVBh0FKfFgIcO++ljWm7D47eYcwDQYJKoZIhvcNAQEL +BQAwdjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwHhcNMjAwODA1MTkxNjAxWhcNMjIw +ODA1MTkxNjAxWjCBqDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsM +EEx5ZnQgRW5naW5lZXJpbmcxGzAZBgNVBAMMElRlc3QgRnJvbnRlbmQgVGVhbTEl +MCMGCSqGSIb3DQEJARYWZnJvbnRlbmQtdGVhbUBseWZ0LmNvbTCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBANarN0opNpHy8zDYi2ih9rLXqgpjVEHSsUDo +eX506Uil9U5I2hCq5XDDsKYhFrRh54G7s9AXCh8orbCBwALPznLwBqSeVy1rnUUR +dWlAGdoCGDbVPC1+Eg7Q8AM8KkGN49x6PcIepHClkX+nnwRogrSXnPHydik9Mhgc ++009IcNf4pPl5U1WcL+JEN+x6iXg+nERoKbVHSp0mkS8CQYdPtx2p+QsnEKRyG3W +eJ7msZPW77th27yxzRYDK4TARQAmHchN3FeF2qs1ak2e2s2chTnYb9qfBb++aanA +fewjnfd/enuvT7Uihswv0dniNTE5I5tDCigXYg1Pp3AKxs7Cja0CAwEAAaOBwTCB +vjAMBgNVHRMBAf8EAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcD +AgYIKwYBBQUHAwEwQgYDVR0RBDswOYYfc3BpZmZlOi8vbHlmdC5jb20vZnJvbnRl +bmQtdGVhbYIIbHlmdC5jb22CDHd3dy5seWZ0LmNvbTAdBgNVHQ4EFgQUxEyk+snc +pQdwv9y8NLXJeJQRP/QwHwYDVR0jBBgwFoAUlkvaLFO0vpXGk3Pip6SfLg1yGIcw +DQYJKoZIhvcNAQELBQADggEBABT1893YgY90gOrkbFz7u81VXa/pxGGIvqoNlFEd +PLP+QSdJVOHL7Ud7qecrUkeJ5Xn57BI4x4lfLVbs74phW8EDspxQ5FRtRQzrQZE9 +R/0VdeAVJIAm+108cfmX83rFriktQC9ffRFnsamCuDjpKWLg/3tMaNSAz7yfxYsJ +sUCRKnLRb7kRyD4tiztHZjY9F1sEYoVbNuQiyFeDZzwLhe9S1WN9rUYA4n+wmyDh +MAjmaJYWwyTZx6SwSevYaXAZSdlxqxVyBrTeNELhbXn0vRxK2J5c9gqXNf2vRKrW +jDsrvfG9XRMBXv1USqplMVduvml5OdTlnALD4Gd1WDJfVt8= -----END CERTIFICATE----- diff --git a/test/config/integration/certs/clientcert_hash.h b/test/config/integration/certs/clientcert_hash.h index cd77a55df7cf..c40b3818f6cc 100644 --- a/test/config/integration/certs/clientcert_hash.h +++ b/test/config/integration/certs/clientcert_hash.h @@ -1,3 +1,3 @@ // NOLINT(namespace-envoy) -constexpr char TEST_CLIENT_CERT_HASH[] = "90:CA:A3:E0:B0:AD:8E:E6:4F:BC:11:6C:7B:E5:9D:35:11:2B:46:" - "71:5F:4D:5C:52:85:37:23:08:38:28:B4:D6"; +constexpr char TEST_CLIENT_CERT_HASH[] = "0E:80:B9:1F:11:3F:FB:43:9B:CB:5A:70:2D:04:C5:00:D6:81:68:" + "8C:CA:F8:91:92:21:CA:0F:91:61:A5:FB:01"; diff --git a/test/config/integration/certs/clientkey.pem b/test/config/integration/certs/clientkey.pem index e1b46d735ec6..aee2284618ee 100644 --- a/test/config/integration/certs/clientkey.pem +++ b/test/config/integration/certs/clientkey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAqLi5RmzH6TleUof821DaOhh84aQdUP1Ksc5HZoHReLc5t8PN -0KpGtv0OEeP14t4Jm1JbM38NgMfKLiGz5ceZv9f8rb1FTnMr5LLvmnvdlj+tp7pC -90Sy7eHLb0ogNIvRXhazZrjPkf3EnTH/bZdtIXi9v24ApiJTa+ovijLGUJVptxkZ -2q34fEVsMs6lQyy07zjvofY8R3ympzdQcCJtEWB0YMX8noXpeGr+rfcbMpROHbRr -5OAzu2jvfCHzbZ+qx8riGNoPFqeDMG7PWfrl0gnPPzUwjpvRkrxtoSUjCHanZxEy -3+Xk4kVAdxReHpd51zYTkxa9IgMnS+2yGKpKawIDAQABAoIBAA1Ivg21cugCBFMr -MdVywDviwbJiYYyG5OKrAyQnBH8krf6yA/px7a9qrTjrYejC4q7ABT5AuqdxE5Ie -RTPKS2i3cMWdKV/L4aDYFdVr+z5hNSMHn04oso3YQVQ52d9JQurNjsJ/upgcCub1 -kM7oJUeFYis4VgS+nyLYBXY0GTku6aheM/+Z0xTEvIc3Cfzb94aCqnGwHe+TvxN0 -zCKhm4bUO4HE5UQzHXO9oxMlOlI7hDAUTSfhT0FpOtxnmisl+XFRSGSvJmNbK4jJ -8dlqIuXsSh4X9NPEnhPz1ieg4+hcUelg5SvSNLNAVQPmm1nNluwYhT41iU+seiiJ -A3uUJEECgYEA2fAS32IQox/8+OpKPipvR13tHek04AsJSGuNiio8qXlQorlXBRYE -esOEEYJyKiNWZ7NXTCAjyDiRp48aM62Fy1ykDR4IbDb8JRAdZpJBCjy5U5BKfZTY -jR3rZOA+8Bivozmtdv+0RGxhffSIGPpB1/4Dgmkw375YLAB4cVffg30CgYEAxjA1 -o8ImIW+9eKTRV2yWUHiUZ4iae5s54nCnz24YqJVLooWtU/TXFKrTiYJs20HBWiG0 -+dsJ27LXd0Y13DDQN/ZT/luvWbhMkFYGQ7Ou7XDXB6ujgl6A5MZ7mKfxIv+kX5QM -F1NTVl3TeUU+6XAYeUf1xmlDjot3sNNv5NdvGgcCgYBRzUnYLP/fqscSSyaY1Oa1 -2+x/mKQvIBVY6H3VCWuBlTaODZE7KHt/9NkilVrytBbfj7JJsZqcsZcCVLVaBly8 -60XsYoR40d6srrLKaEUfaZGKaxN6tZ7ewQc08vLMvgdW9fRFQU9Ri3jAhUN8VJrY -TtDUZ1Vf9hs0UOzkZj5QJQKBgCfS7iRe0eysGGWSsOIhVr8Ky79WKrylv2bp/j5n -QBs4DL+2ntKdA08K2IDsLVWNi/3Bgi0mv39fG37DI/V/9YcZP12ALOcZaoEiWBXo -mEDsCLlo2u1KchoGbDWLoZ/HwM7X3+ob+0YCioj2yiJ8PN66AAADjOiqy71Db1uL -kq6nAoGAN8dAwrJFQ2mQYKFQrbWt3/x+YHCAvkqIFdVWn9+3/fWZ8Rcwctl/mau/ -7dL5tU/s3wO8KkOH2PTjvtUORZR+e/FGq7hmXVV1lnl7GBENW6THN08xq9e/4Clm -koYsIpn/e+t2AVR1UENvv9sKwAsDV1yqHwRfUSFtg/qtj7103YY= +MIIEowIBAAKCAQEA1qs3Sik2kfLzMNiLaKH2steqCmNUQdKxQOh5fnTpSKX1Tkja +EKrlcMOwpiEWtGHngbuz0BcKHyitsIHAAs/OcvAGpJ5XLWudRRF1aUAZ2gIYNtU8 +LX4SDtDwAzwqQY3j3Ho9wh6kcKWRf6efBGiCtJec8fJ2KT0yGBz7TT0hw1/ik+Xl +TVZwv4kQ37HqJeD6cRGgptUdKnSaRLwJBh0+3Han5CycQpHIbdZ4nuaxk9bvu2Hb +vLHNFgMrhMBFACYdyE3cV4XaqzVqTZ7azZyFOdhv2p8Fv75pqcB97COd9396e69P +tSKGzC/R2eI1MTkjm0MKKBdiDU+ncArGzsKNrQIDAQABAoIBABd8T+Y7MA8zp0uW +xVnDLnxOf/n2+AbjiCTzyib9n3AlR/symTjtmYCGyFLEl/lQJMXaxUdk3eSezLHc +4CbumUWV4QQtlpgPh/tAd7n2G13wkLmfBqBrhIo+baPM90qIvX8nmI4eUBtK4eo3 +anxO+s3LMI5/2lGUsmBU+2Ft6L25En1JpUg3THo0/Ek8VdySQ9V1PTFpTjKsd6Na +DxKYpM85GE1mOKowQqAdbvFByqXoCZcd71ZE9t5XoohDLgPPHq7MA4Z0mKOK5qVG +IR0pBThm4Ij7IFLjzV1b9IGWiBKj9uWPD5VNSad0lrMFoXBcGsp6YblrfpOkZv41 +PALL8rkCgYEA8a9aYsIvOYnE82MCut2xuTWE7lHFpxDV3rJqN51RZipgpWgz76PM +kuKcqp3Fi813hz5abUOizQAaa2d57AmJ+VI0i6CbbCMcwu5javYuwOC7b47WeGIy +fclCgUhbns/vZVatb1pJ+kd5sagT80GehlGoHevd0erzSIbcgZ0KMH8CgYEA42I4 +J9wJ14pC/aGCTQbpne5RjAi07dpmTwlwpsFIHmUuqaJe25eq+R5lnKLqTyGbxDUQ +6aU/8i+G4631gMYng7kyuashBTLsFoyMixLWApDO/U1Lil2dDOXM+DqPKuLdyq6v +SYbfuh87J5BEfmtfFwS5sVSIHNcPp7nUJQ8o69MCgYBv9tV/tQgdtsZoHrFQEo5Y +CAQ6R+WyPOlnju4IL7hbBTzaxAhzd0W5soPzwr2Ww6whGnDX96J/KBIVOc3Q3KZv +u3aeTNxT33xejgO+tKf6MOKEjv6qrItJnKhTrkrLqvbz0pDsaj6lVOF6vSvo4Lho +74Fbwz5zFk54hgm3fiIPTwKBgQCNrrHXBGCNkXVUnMKYRGplIg5l9zblzmRZc0Ri +Y6UQa3O795Srt8GtIKeoBkuBqytoArjbHUDPI5YlYEvNRatxhIB6+IrGtogtNL6O +GdqIFrsjUnpzaQlm8/nX4oU678nLdTV71zKowrUVXeuP6k+CBEvAly+I6Oi0VjI+ +NUgGSQKBgG0qfjZ+k6KaLEvw/Ry6j1TZEzR9PNryIFb3zmcalZakZ92C9xP6+Wyw +KaZsJt1b7kOZ/VdKrlUzfCEe8+CFy8eBWKuffb5v6QTa5af3+igGpjwygbjLW3Aw +Jq/Ykllf3iQP/XSq6fQxvhJ4zYYjnNvLAY9yV+MLJkkqaQTnoKSq -----END RSA PRIVATE KEY----- diff --git a/test/config/integration/certs/server_ecdsacert.pem b/test/config/integration/certs/server_ecdsacert.pem index d63b3852359f..6a0c863d41d9 100644 --- a/test/config/integration/certs/server_ecdsacert.pem +++ b/test/config/integration/certs/server_ecdsacert.pem @@ -1,22 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDljCCAn6gAwIBAgIJAILStmLgUUcWMA0GCSqGSIb3DQEBCwUAMHYxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2luZWVyaW5nMRAw -DgYDVQQDDAdUZXN0IENBMB4XDTE4MTIxNzIwMTgwMFoXDTIwMTIxNjIwMTgwMFow -gaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1T -YW4gRnJhbmNpc2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2lu -ZWVyaW5nMRowGAYDVQQDDBFUZXN0IEJhY2tlbmQgVGVhbTEkMCIGCSqGSIb3DQEJ -ARYVYmFja2VuZC10ZWFtQGx5ZnQuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcD -QgAEgVE4DzGyifdO4yefn1bEhFfkTrUjfV3Lay9NQz6D1SosrCnk7KZtl6Nbc/U6 -KiGlN+9GpD0ulKEljgjn7TTtT6OBwDCBvTAMBgNVHRMBAf8EAjAAMAsGA1UdDwQE -AwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwQQYDVR0RBDowOIYe -c3BpZmZlOi8vbHlmdC5jb20vYmFja2VuZC10ZWFtgghseWZ0LmNvbYIMd3d3Lmx5 -ZnQuY29tMB0GA1UdDgQWBBQDO72plAAjzF3GWFAC+8fyzUpl0zAfBgNVHSMEGDAW -gBQUM9b2kmz7njy/vuxkzKiwDLZN5DANBgkqhkiG9w0BAQsFAAOCAQEAppJXn2t3 -7qER7Dj4HjNq3GdASgiLsCgjus5Mbfje4rGbb73WJnsMVpOZtWwC4KyIxDHBzFmR -ntS3g4eJ3FxFkr5ls+RDzSAyIJBzh/Bwt1Yrkn3Yh3HxgsYy6Rd7z4+hHgaqnrza -ESdmWHoMGg7AJy17drIDmxAK7Os5qObWxytsrUL+lTIBJ3OS2IDiB2tT/WrRMSx9 -umYff6YsGLsOGtVgRsc640wKex7r468p1tkwj5r2PxvH9V9L3hp9QuH+W4gR6IYe -HhcCTllp+3iH0vmcXCZRP984nnwRBPP3tBuT+/zmUyDp0UPBHqF+jcHxMB/YA9An -SziyfTA1g4CzRA== +MIIDoTCCAomgAwIBAgIUJuVBh0FKfFgIcO++ljWm7D47eYYwDQYJKoZIhvcNAQEL +BQAwdjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwHhcNMjAwODA1MTkxNjAxWhcNMjIw +ODA1MTkxNjAxWjCBpjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsM +EEx5ZnQgRW5naW5lZXJpbmcxGjAYBgNVBAMMEVRlc3QgQmFja2VuZCBUZWFtMSQw +IgYJKoZIhvcNAQkBFhViYWNrZW5kLXRlYW1AbHlmdC5jb20wWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAARbRc4+r8sbKLs89/fS0Y8CSTVNKn8opkCFx/UyLPvELLn6 +lQ86GjcQIqJRqoEqEwP7iyCaDhFigsOKsNQSzE9/o4HAMIG9MAwGA1UdEwEB/wQC +MAAwCwYDVR0PBAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATBB +BgNVHREEOjA4hh5zcGlmZmU6Ly9seWZ0LmNvbS9iYWNrZW5kLXRlYW2CCGx5ZnQu +Y29tggx3d3cubHlmdC5jb20wHQYDVR0OBBYEFM6Jqpu+mo5ATAcIizKPrFH1qDLK +MB8GA1UdIwQYMBaAFJZL2ixTtL6VxpNz4qekny4NchiHMA0GCSqGSIb3DQEBCwUA +A4IBAQAYGH/EjPHxtE5fQ7kDsRGcafMLqcx6OYhZlqDfMqepbFEWlkL/kuNVrjez +NoG67cc3DSagyVocjr7yCz/kkDm4HowRapGK6eSK3dWhjrQzp2F6ImAAfht7YkhF +JKkuRYULKpsRRj+YVpBpYkUjRVaqLNJJpXOItpvZMsyXDZn/ihy1QerDXaetWXgP +yFRL+g9f+e8OydXgd/lrKByLBCszNPeg4nhGsgZZ1WHj5sl2EY27eAIWRXqFIQbr +VwbLfpFm2R2nrJT8zJF54OUpG/3hKOEkgZ3gSK9+tnQeSM01lXHd1ZzrmzCdItDQ +MNl4mwlDTJLwL/vwMld9UFz0qAYi -----END CERTIFICATE----- diff --git a/test/config/integration/certs/server_ecdsacert_hash.h b/test/config/integration/certs/server_ecdsacert_hash.h index 309feff18643..2b828070d615 100644 --- a/test/config/integration/certs/server_ecdsacert_hash.h +++ b/test/config/integration/certs/server_ecdsacert_hash.h @@ -1,3 +1,3 @@ // NOLINT(namespace-envoy) -constexpr char TEST_SERVER_ECDSA_CERT_HASH[] = "95:9A:D0:18:52:40:D9:88:35:B9:52:96:C1:50:86:82:76:" - "FD:24:AD:48:F9:8F:0C:13:36:BA:2D:36:F2:4C:F3"; +constexpr char TEST_SERVER_ECDSA_CERT_HASH[] = "B1:A4:B5:39:F8:91:77:00:3E:3E:23:1D:F2:AD:78:34:85:" + "D3:F0:D4:EF:D2:88:EA:B5:92:F7:71:E1:B8:0D:F5"; diff --git a/test/config/integration/certs/server_ecdsakey.pem b/test/config/integration/certs/server_ecdsakey.pem index a46830f1a653..df1f17d826d1 100644 --- a/test/config/integration/certs/server_ecdsakey.pem +++ b/test/config/integration/certs/server_ecdsakey.pem @@ -2,7 +2,7 @@ BggqhkjOPQMBBw== -----END EC PARAMETERS----- -----BEGIN EC PRIVATE KEY----- -MHcCAQEEICqemTBghfUmDlTHDq5GCz5CV51QLvQEkUGOxhtOR6hboAoGCCqGSM49 -AwEHoUQDQgAEgVE4DzGyifdO4yefn1bEhFfkTrUjfV3Lay9NQz6D1SosrCnk7KZt -l6Nbc/U6KiGlN+9GpD0ulKEljgjn7TTtTw== +MHcCAQEEIGY3X0at4PZN6X+MHnEC/vwjaKTq/ffwp0KfMcAKCwgToAoGCCqGSM49 +AwEHoUQDQgAEW0XOPq/LGyi7PPf30tGPAkk1TSp/KKZAhcf1Miz7xCy5+pUPOho3 +ECKiUaqBKhMD+4sgmg4RYoLDirDUEsxPfw== -----END EC PRIVATE KEY----- diff --git a/test/config/integration/certs/servercert.pem b/test/config/integration/certs/servercert.pem index f65b3b34ae7c..c654495573e4 100644 --- a/test/config/integration/certs/servercert.pem +++ b/test/config/integration/certs/servercert.pem @@ -1,26 +1,26 @@ -----BEGIN CERTIFICATE----- -MIIEYTCCA0mgAwIBAgIJAILStmLgUUcVMA0GCSqGSIb3DQEBCwUAMHYxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2luZWVyaW5nMRAw -DgYDVQQDDAdUZXN0IENBMB4XDTE4MTIxNzIwMTgwMFoXDTIwMTIxNjIwMTgwMFow -gaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1T -YW4gRnJhbmNpc2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2lu -ZWVyaW5nMRowGAYDVQQDDBFUZXN0IEJhY2tlbmQgVGVhbTEkMCIGCSqGSIb3DQEJ -ARYVYmFja2VuZC10ZWFtQGx5ZnQuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAuvPdQdmwZongPAgQho/Vipd3PZWrQ6BKxIb4l/RvqtVP321IUTLs -4vVwpXoYJ+12L+XOO3jCInszs53tHjFpTI1GE8/sasmgR6LRr2krwSoVRHPqUoc9 -tzkDG1SzKP2TRTi1MTI3FO+TnLFahntO9Zstxhv1Epz5GZ/xQLE0/LLoRYzcynL/ -iflk18iL1KM8i0Hy4cKjclOaUdnh2nh753iJfxCSb5wJfx4FH1qverYHHT6FopYR -V40Cg0yYXcYo8yNwrg+EBY8QAT2JOMDokXNKbZpmVKiBlh0QYMX6BBiW249v3sYl -3Ve+fZvCkle3W0xP0xJw8PdX0NRbvGOrBQIDAQABo4HAMIG9MAwGA1UdEwEB/wQC -MAAwCwYDVR0PBAQDAgXgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATBB -BgNVHREEOjA4hh5zcGlmZmU6Ly9seWZ0LmNvbS9iYWNrZW5kLXRlYW2CCGx5ZnQu -Y29tggx3d3cubHlmdC5jb20wHQYDVR0OBBYEFLHmMm0DV9jCHJSWVRwyPYpBw62r -MB8GA1UdIwQYMBaAFBQz1vaSbPuePL++7GTMqLAMtk3kMA0GCSqGSIb3DQEBCwUA -A4IBAQAwx3/M2o00W8GlQ3OT4y/hQGb5K2aytxx8QeSmJaaZTJbvaHhe0x3/fLgq -uWrW3WEWFtwasilySjOrFOtB9UNmJmNOHSJD3Bslbv5htRaWnoFPCXdwZtVMdoTq -IHIQqLoos/xj3kVD5sJSYySrveMeKaeUILTkb5ZubSivye1X2yiJLR7AtuwuiMio -CdIOqhn6xJqYhT7z0IhdKpLNPk4w1tBZSKOXqzrXS4uoJgTC67hWslWWZ2VC6IvZ -FmKuuGZamCCj6F1QF2IjMVM8evl84hEnN0ajdkA/QWnil9kcWvBm15Ho+oTvvJ7s -M8MD3RDSq/90FSiME4vbyNEyTmj0 +MIIEbDCCA1SgAwIBAgIUJuVBh0FKfFgIcO++ljWm7D47eYUwDQYJKoZIhvcNAQEL +BQAwdjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwHhcNMjAwODA1MTkxNjAxWhcNMjIw +ODA1MTkxNjAxWjCBpjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsM +EEx5ZnQgRW5naW5lZXJpbmcxGjAYBgNVBAMMEVRlc3QgQmFja2VuZCBUZWFtMSQw +IgYJKoZIhvcNAQkBFhViYWNrZW5kLXRlYW1AbHlmdC5jb20wggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQC9JgaI7hxjPM0tsUna/QmivBdKbCrLnLW9Teak +RH/Ebg68ovyvrRIlybDT6XhKi+iVpzVY9kqxhGHgrFDgGLBakVMiYJ5EjIgHfoo4 +UUAHwIYbunJluYCgANzpprBsvTC/yFYDVMqUrjvwHsoYYVm36io994k9+t813b70 +o0l7/PraBsKkz8NcY2V2mrd/yHn/0HAhv3hl6iiJme9yURuDYQrae2ACSrQtsbel +KwdZ/Re71Z1awz0OQmAjMa2HuCop+Q/1QLnqBekT5+DH1qKUzJ3Jkq6NRkERXOpi +87j04rtCBteCogrO67qnuBZ2lH3jYEMb+lQdLkyNMLltBSdLAgMBAAGjgcAwgb0w +DAYDVR0TAQH/BAIwADALBgNVHQ8EBAMCBeAwHQYDVR0lBBYwFAYIKwYBBQUHAwIG +CCsGAQUFBwMBMEEGA1UdEQQ6MDiGHnNwaWZmZTovL2x5ZnQuY29tL2JhY2tlbmQt +dGVhbYIIbHlmdC5jb22CDHd3dy5seWZ0LmNvbTAdBgNVHQ4EFgQU2XcTZbc0xKZf +gNVKSvAbMZJCBoYwHwYDVR0jBBgwFoAUlkvaLFO0vpXGk3Pip6SfLg1yGIcwDQYJ +KoZIhvcNAQELBQADggEBAFW05aca3hSiEz/g593GAV3XP4lI5kYUjGjbPSy/HmLr +rdv/u3bGfacywAPo7yld+arMzd35tIYEqnhoq0+/OxPeyhwZXVVUatg5Oknut5Zv +2+8l+mVW+8oFCXRqr2gwc8Xt4ByYN+HaNUYfoucnjDplOPukkfSuRhbxqnkhA14v +Lri2EbISX14sXf2VQ9I0dkm1hXUxiO0LlA1Z7tvJac9zPSoa6Oljke4D1iH2jzwF +Yn7S/gGvVQgkTmWrs3S3TGyBDi4GTDhCF1R+ESvXz8z4UW1MrCSdYUXbRtsT7sbE +CjlFYuUyxCi1oe3IHCeXVDo/bmzwGQPDuF3WaDNSYWU= -----END CERTIFICATE----- diff --git a/test/config/integration/certs/servercert_hash.h b/test/config/integration/certs/servercert_hash.h index 6e83ce970527..b75188ddab54 100644 --- a/test/config/integration/certs/servercert_hash.h +++ b/test/config/integration/certs/servercert_hash.h @@ -1,3 +1,3 @@ // NOLINT(namespace-envoy) -constexpr char TEST_SERVER_CERT_HASH[] = "D3:F5:97:8F:8F:7E:CA:63:C3:FF:61:89:95:D7:47:31:E3:BA:9B:" - "3C:44:10:A1:57:61:B7:9F:4D:65:90:F8:7F"; +constexpr char TEST_SERVER_CERT_HASH[] = "E6:D4:55:6E:BC:91:E2:2F:1E:42:48:05:3C:3E:B7:97:ED:3E:51:" + "E3:A9:C4:87:1B:FF:61:76:25:6B:F3:95:7C"; diff --git a/test/config/integration/certs/serverkey.pem b/test/config/integration/certs/serverkey.pem index 6870eed5bfc6..27ea60a966ea 100644 --- a/test/config/integration/certs/serverkey.pem +++ b/test/config/integration/certs/serverkey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAuvPdQdmwZongPAgQho/Vipd3PZWrQ6BKxIb4l/RvqtVP321I -UTLs4vVwpXoYJ+12L+XOO3jCInszs53tHjFpTI1GE8/sasmgR6LRr2krwSoVRHPq -Uoc9tzkDG1SzKP2TRTi1MTI3FO+TnLFahntO9Zstxhv1Epz5GZ/xQLE0/LLoRYzc -ynL/iflk18iL1KM8i0Hy4cKjclOaUdnh2nh753iJfxCSb5wJfx4FH1qverYHHT6F -opYRV40Cg0yYXcYo8yNwrg+EBY8QAT2JOMDokXNKbZpmVKiBlh0QYMX6BBiW249v -3sYl3Ve+fZvCkle3W0xP0xJw8PdX0NRbvGOrBQIDAQABAoIBAQCkPLR1sy47BokN -c/BApn9sn5/LZH7ujBTjDce6hqzLIVZn6/OKEfj1cbWiSd6KxRv8/B/vMykpbZ5/ -/w9eZP4imEGmChWhwruh8zHOrdAYhEXmuwZxtgnLurQ2AHTcX9hPCYB0Va76H3ZI -Q65JUm6NaeQOlGT6ExjrIA2rTYJFM84I1xH3XbDulS9S2FXNP9RIjV70HzvZw2LR -1qSNfrnGAEbUCdrZT4BAYTGam5L061ofencYLAorr8K0eVWhUjGV9Jjpq8aG8zy5 -Oy1070I0d7Iexfu7T1sQDIqpNkOtQxI8feQEKeKlRKYx6YEQ9vaVwBGa0SBVxQem -E3YdXBnBAoGBAORlz8wlYqCx25htO/eLgr9hN+eKNhNTo4l905aZrG8SPinaHl15 -n+dQdzlJMVm/rh5+VE0NR0U/vzd3SrdnzczksuGFn0Us/Yg+zOl1+8+GFAtqw3js -udFLKksChz4Rk/fZo2djtSiFS5aGBtw0Z9T7eorubkTSSfJ7IT99HIu5AoGBANGL -0ff5U2LV/Y/opKP7xOlxSCVI617N5i0sYMJ9EUaWzvquidzM46T4fwlAeIvAtks7 -ACO1cRPuWredZ/gEZ3RguZMxs6llwxwVCaQk/2vbOfATWmyqpGC9UBS/TpYVXbL5 -WUMsdBs4DdAFz8aCrrFBcDeCg4V4w+gHYkFV+LetAoGAB3Ny1fwaPZfPzCc0H51D -hK7NPhZ6MSM3YJLkRjN5Np5nvMHK383J86fiW9IRdBYWvhPs+B6Ixq+Ps2WG4HjY -c+i6FTVgvsb69mjmEm+w6VI8cSroeZdvcG59ULkiZFn6c8l71TGhhVLj5mM08hYb -lQ0nMEUa/8/Ebc6qhQG13rECgYEAm8AZaP9hA22a8oQxG9HfIsSYo1331JemJp19 -rhHX7WfaoGlq/zsrWUt64R2SfA3ZcUGBcQlD61SXCTNuO+LKIq5iQQ4IRDjnNNBO -QjtdvoVMIy2/YFXVqDIOe91WRCfNZWIA/vTjt/eKDLzFGv+3aPkCt7/CkkqZErWq -SnXkUGECgYAvkemYu01V1WcJotvLKkVG68jwjMq7jURpbn8oQVlFR8zEh+2UipLB -OmrNZjmdrhQe+4rzs9XCLE/EZsn7SsygwMyVhgCYzWc/SswADq7Wdbigpmrs+grW -fg7yxbPGinTyraMd0x3Ty924LLscoJMWUBl7qGeQ2iUdnELmZgLN2Q== +MIIEpAIBAAKCAQEAvSYGiO4cYzzNLbFJ2v0JorwXSmwqy5y1vU3mpER/xG4OvKL8 +r60SJcmw0+l4Sovolac1WPZKsYRh4KxQ4BiwWpFTImCeRIyIB36KOFFAB8CGG7py +ZbmAoADc6aawbL0wv8hWA1TKlK478B7KGGFZt+oqPfeJPfrfNd2+9KNJe/z62gbC +pM/DXGNldpq3f8h5/9BwIb94ZeooiZnvclEbg2EK2ntgAkq0LbG3pSsHWf0Xu9Wd +WsM9DkJgIzGth7gqKfkP9UC56gXpE+fgx9ailMydyZKujUZBEVzqYvO49OK7QgbX +gqIKzuu6p7gWdpR942BDG/pUHS5MjTC5bQUnSwIDAQABAoIBADEMwlcSAFSPuNln +hzJ9udj0k8md4T8p5Usw/2WLyeJDdBjg30wjQniAJBXgDmyueWMNmFz4iYgdP1CG +/vYOEPV7iCZ7Da/TDZd77hYKo+MevuhD4lSU1VEoyCDjNA8OxKyHJB77BwmlYS+0 +nE3UOPLji47EOVfUTbvnRBSmn3DCSHkQiRIUP1xMivoiZgKJn+D+FxSMwwiq2pQR +5tdo7nh2A8RxlYUbaD6i4poUB26HVm8vthXahNEkLpXQOz8MWRzs6xOdDHRzi9kT +ItRLa4A/3LIATqviQ2EpwcALHXcULcNUMTHORC1EHPvheWR5nLuRllYzN4ReoeHC +3+A5KEkCgYEA52rlh/22/rLckCWugjyJic17vkg46feSOGhjuP2LelrIxNlg491y +o28n8lQPSVnEp3/sT7Y3quVvdboq4DC9LTzq52f6/mCYh9UQRpljuSmFqC2MPG46 +Zl5KLEVLzhjC8aTWkhVINSpz9vauXderOpFYlPW32lnRTjJWE276kj8CgYEA0T2t +ULnn7TBvRSpmeWzEBA5FFo2QYkYvwrcVe0pfUltV6pf05xUmMXYFjpezSTEmPhh6 ++dZdhwxDk+6j8Oo61rTWucDsIqMj5ZT1hPNph8yQtb5LRlRbLGVrirU9Tp7xTgMq +3uRA2Eka1d98dDBsEbMIVFSZ2MX3iezSGRL6j/UCgYEAxZQ82HjEDn2DVwb1EXjC +LQdliTZ8cTXQf5yQ19aRiSuNkpPN536ga+1xe7JNQuEDx8auafg3Ww98tFT4WmUC +f2ctX9klMJ4kXISK2twHioVq+gW5X7b04YXLajTX3eTCPDHyiNLmzY2raMWAZdrG +9MA3kyafjCt3Sn4rg3gTM10CgYEAtJ8WRpJEd8aQttcUIItYZdvfnclUMtE9l0su +GwCnalN3xguol/X0w0uLHn0rgeoQhhfhyFtY3yQiDcg58tRvODphBXZZIMlNSnic +vEjW9ygKXyjGmA5nqdpezB0JsB2aVep8Dm5g35Ozu52xNCc8ksbGUO265Jp3xbMN +5iEw9CUCgYBmfoPnJwzA5S1zMIqESUdVH6p3UwHU/+XTY6JHAnEVsE+BuLe3ioi7 +6dU4rFd845MCkunBlASLV8MmMbod9xU0vTVHPtmANaUCPxwUIxXQket09t19Dzg7 +A23sE+5myXtcfz6YrPhbLkijV4Nd7fmecodwDckvpBaWTMrv52/Www== -----END RSA PRIVATE KEY----- diff --git a/test/config/integration/certs/upstreamcacert.pem b/test/config/integration/certs/upstreamcacert.pem index c3d5692354bd..f3f76898c347 100644 --- a/test/config/integration/certs/upstreamcacert.pem +++ b/test/config/integration/certs/upstreamcacert.pem @@ -1,24 +1,24 @@ -----BEGIN CERTIFICATE----- -MIID7zCCAtegAwIBAgIUQygBeIE4nv9JGaDKixnhwkK5viEwDQYJKoZIhvcNAQEL +MIID7zCCAtegAwIBAgIUTQZdxxw6y4+Te1kv8hDza/KXTHUwDQYJKoZIhvcNAQEL BQAwfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n -aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzA4MjE0 -NTU2WhcNMjEwNzA3MjE0NTU2WjB/MQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2Fs +aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMjAwODA1MTkx +NjAyWhcNMjIwODA1MTkxNjAyWjB/MQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2Fs aWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzENMAsGA1UECgwETHlmdDEZ MBcGA1UECwwQTHlmdCBFbmdpbmVlcmluZzEZMBcGA1UEAwwQVGVzdCBVcHN0cmVh -bSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMJ7AetbhOCUxB/A -yYt+4rxyMVUFX9izqbOU9nuUxsB/avGhYpVjj5cNaLPdGX+c7g65Vz0yGDSskDGD -ukcSFqRSZ2E4/S4gKSIMEslBr2OX+Dqh0XmoAwl4IrtZefCE3inivJdzm0JwI7Yr -k2qQqsTpJnsWkMSxXUQJYTJ56UFXTkKqF3jSReIQtFMV65T/2x2NLRJ8KuMS7Mbo -BTBATRsUfbJJWCnzcp2LrKV5sZ/HsJLK/F74jdcvfJQMW49Lq1TZaB5NYSVyFEf6 -tiT43JOcvVkRPBgHDtaiDhWF2WTmPSEB6cHaRwGgBFwjQ1SvZR6f6xexocn44GZE -oSqWJN8CAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw -HQYDVR0OBBYEFOLTMLryzNAcuxe3cKEhClkL7IduMB8GA1UdIwQYMBaAFOLTMLry -zNAcuxe3cKEhClkL7IduMA0GCSqGSIb3DQEBCwUAA4IBAQBT88sT8RsoAk6PnnVs -KWBoC75BnIZr8o1nxBK0zog6Ez4J32aVzEXPicgBg6hf6v77eqbbQ+O7Ayf+YQWj -l9w9IiXRW1x94tKBrX44O85qTr/xtkfpmQWGKq5fBpdJnZp7lSfGfaG9gasPUNpG -gfvF/vlYrrJoyvUOG6HQjZ7n7m6f8GEUymCtC68oJcLVL0xkvx/jcvGeJfI5U6yr -z9nc1W7FcOhrFEetOIH2BwlIN5To3vPbN4zEzt9VPUHZ3m2899hUiMZJaanEexp7 -TZJJ12rHSIJ4MKwQQ5fEmioeluM0uY7EIR72VEsudA8bkXSkbDGs6Q49K9OX+nRB -4P3c +bSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOFT8hbqRn+9AKU2 +IFtZKFFYpt7v2x1e8gtzgPm3TT7RJcV2GLeT1cOwubL81ArQmwfyVlwJkt1wK7Uw ++Z4FvtcCjQc4dR3yxkIdhzZOiq7PbQgAjyRNNGmneYTAvpXwC+l8ZV2M66ihUKgj +7iGiqQCvYhuYIb7BEnOj20nFuvHlxaDWOst4SQgZmRIkQyA8rrAIRfu7aQiCEla5 +86AXcXV4gmOW3dsKNoXO8Fr+9mtAmJKocLtlUkCeDW+WYqv6RLjMVa915khNQLde +bL+5hYxBcKYB10wOVzSTCfM6fbqtpqJZEdlGjkKtQ2Szy3mpoAJKPmZYzodVhL6N +LhoLjZ8CAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +HQYDVR0OBBYEFDtmHVOikybtJjVEI4Q7wvUbwgBkMB8GA1UdIwQYMBaAFDtmHVOi +kybtJjVEI4Q7wvUbwgBkMA0GCSqGSIb3DQEBCwUAA4IBAQAT3kBm2uCpB4cAmdgu +u6sqxUvYFzYlHFnWrQ3ZFwMrLRSzUdrcp2nSQz+e8VeXI2SkLPCD5Xg+8GGLWA5X +lH6tvVx41cRqSr611ebxPVWkEeP+ALkHo4xUbcR5WUJD52VxzqYbhavYFjB2FzqA +OfefKyXIhcKtezKBwaJbVn9FseH49q6UNjYODOY88rW+2mvDoZWBUuti8CxNhIiu +RHnGimY7H565NpbPliVlo2GhiKhJvyPwK7+cjfj68HaoixlXHmrg506bczO/Gt1a +USQmjtB05h8bki0LQDiCQu1fdOPEflJnv3VdFz2SSKNRab2asP+KbRPURUW8f9zN +GNxR -----END CERTIFICATE----- diff --git a/test/config/integration/certs/upstreamcakey.pem b/test/config/integration/certs/upstreamcakey.pem index 2fe99b9c2c10..08b5a9bfcffa 100644 --- a/test/config/integration/certs/upstreamcakey.pem +++ b/test/config/integration/certs/upstreamcakey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAwnsB61uE4JTEH8DJi37ivHIxVQVf2LOps5T2e5TGwH9q8aFi -lWOPlw1os90Zf5zuDrlXPTIYNKyQMYO6RxIWpFJnYTj9LiApIgwSyUGvY5f4OqHR -eagDCXgiu1l58ITeKeK8l3ObQnAjtiuTapCqxOkmexaQxLFdRAlhMnnpQVdOQqoX -eNJF4hC0UxXrlP/bHY0tEnwq4xLsxugFMEBNGxR9sklYKfNynYuspXmxn8ewksr8 -XviN1y98lAxbj0urVNloHk1hJXIUR/q2JPjck5y9WRE8GAcO1qIOFYXZZOY9IQHp -wdpHAaAEXCNDVK9lHp/rF7GhyfjgZkShKpYk3wIDAQABAoIBAD+CoBvWJUyaCHo+ -IRNW+oCD4ixbtvMzqOWmbd/ptAZFFg2WoHUcsFWp4VlriNoty2gvipfHdjQtbmFd -HUX8WDyNVIlhbPzVL9mYi8IBm18wz7WGBrxt65/6BY2dKL8tBMg07VWgQUGvEVp6 -XIfeeoYXhaOIuPoi2cxQK9eqDExzvb5AA1AS+FbYcKF1ma2Kb/mO52OQAsPmPnul -yyosInO2PFdNqlvYd5qOfJdPF1747nn4taigH1CKdDZ86GNufShWvcdiR/uYL/Ln -vu4Um7Ha05AFl9p+7TPqyuE1+nH1nKOqP8++C5TkDqLhPyzDUxENU50eCpQHhNiK -Jrvt1VECgYEA+k5E+pyg+Ji4XQnwNMbP2P8jgnFDSL7HfHuE3NfdomvoDit78LFw -/FzosBpv79lSXh8wplBIs5UwrAqaWoV0GQWoySM27hRDM5/c0xhqWS2c6gMGeUup -Tn2YvuXTmi1OsIPayTzQ92GeT3Xg+ojLZdqtLmkEJzAtUndww1HlLIUCgYEAxuef -fXXMfEYCrdEA1cvFGilVxnJXHjzHnky5RUrglwV6wkgfwE18dr4cmOgBte69shvc -8TS6I+KFffelKjSzm7OAEWEL+pGHKK0ALTBBXUJ3qa0PYSrgbCD1/nI3Q08JLSVV -+Xo5kmIGyLJMsLGkH/CSZCNUj450WDmfdcGrqxMCgYACDwC8OuuL/92MTleeZ4Aw -HbESEpJmF8OWP4HROylEe7S14R+s1BjEypLTV/RRuazWv1TsGT7v0ytKTvAEDJLu -3cAMn3CFNr9yvj7XsZy2TQy8U/gKqVekIJ5P+53o57R8+SikfQ6O6kueBa8rAFMD -7G9+MTjqhZfp1Lels5e57QKBgD4+q9WaMKTPT/VPC6DcRNE8EECq9YJb6OgsAGqj -1QbNyy3TXkRSu1l5gv+C00446Ro8x/af1oR2VeomvoQnu/FEyhYmNZZzRkW/Zee+ -SyZBL6tkogR5Y4PTCMhYu9yPdkKvhWkuC6g4jwDtczx0SvVH1rgJqmPGY7hcR/+U -3QELAoGBANxKIoJhDSecjGmjdHBGGmtXHsBZID033Qq6LEPStcHb7aMNdSYCIjZA -FpfqNYPywrqPOjlUVzM2Erz3gmdd5o3OxgbTkSjJPhndSvw9fCU29Oy0PP7qTXgE -Ksfuj92ATYeT+wwWZJ5kfhMjmvhPKhOAdi9au27y5tiO5upDeReh +MIIEpAIBAAKCAQEA4VPyFupGf70ApTYgW1koUVim3u/bHV7yC3OA+bdNPtElxXYY +t5PVw7C5svzUCtCbB/JWXAmS3XArtTD5ngW+1wKNBzh1HfLGQh2HNk6Krs9tCACP +JE00aad5hMC+lfAL6XxlXYzrqKFQqCPuIaKpAK9iG5ghvsESc6PbScW68eXFoNY6 +y3hJCBmZEiRDIDyusAhF+7tpCIISVrnzoBdxdXiCY5bd2wo2hc7wWv72a0CYkqhw +u2VSQJ4Nb5Ziq/pEuMxVr3XmSE1At15sv7mFjEFwpgHXTA5XNJMJ8zp9uq2molkR +2UaOQq1DZLPLeamgAko+ZljOh1WEvo0uGguNnwIDAQABAoIBAFiIeThTuHt8MYK4 +b6I0t8iugnJZ38f8hDHHokd7pBgoaSTar/+BUJ5hE7Wl7VKKgD9xEkl7YX8sEaBR +q+JQ85jbYboSjsHDn+5eV8AYwBjLW1WnkpZ61zskGHT2nmufM677t4A4XGeXam+G +HoyMssaYIn4hGjEu/yb8nK6xyDA+kHuuDthhM9qnrOKpZGRscXY3yolJhscif2jW +Ox2VLomYtlaKcZ1mrBapaLpY12WoRU/3YXvPpvWKpajkWVcqHZfQ56KATLmlLuCJ +aW4+A8gXRPepva2Enc6fXKKpiFiqGlBZ9kKVORGVGndpZLSMxP20SMTovtAz3wxQ +c3kCk0ECgYEA/6Cv7NtfqXLz2ovNIarObFYVuLyN49ubW/JntuRcQA9p5KyQcf8Y ++4LmR3F+te/Cz+QfC7zDgObpxMqvCs3fKLUg1ZLFRtSXtGRiulzhVRrxDk1FU6KO +m2+s2FGKyxB+czxg1jglASOXMRjKJ/K+/eiditcjaYeVzFfC7z9nyRECgYEA4af2 +AJEcvKdYc6PibIpmRGSNhCY5MPE88Rg19qA/0I7s1aytimHBYEKAUTpzcw+gbmNz +IWYYMkpbq8SZD0IGnzTRJTXsOKy8ZxrjlBZMbYsm77qYjTarObLps2Qa/8VharCW +9PAsT8rwA5PfVoTQaVQFHlyXC4266nx3z4gfa68CgYEAsXIkzQFXPXQLbHjBM46y +7icvuuZAhJxsEv6JGj8Y/mr0sgVL26Yd/HFYUt2o/Lhrfg43stkcyT0Bp1ae/Zv9 +Pe/F1BunD80BZfqNQhq5XG9wR+JBrpXX8nQqAptQAjf33xxZiDq/DTRcfntb0TFD +fVPdEITZEydIR+nf6l4UOFECgYEA1AnNwS6aQDNHjDI8+xz5h96sk7aPGwwz5aCI +ZJykGkeTCB1gXJ4K5XbXuHwiK8ZNTC0q7AFRT0BL75Wm9Y1nR4aL2FlZBNBboM7F +dkuVuYF+Ltm5q0fpkSgrLaQtMpW4OlaBItvj536cFeCHhnb6l16aCLOcQwEE2H3o +3xvb2oUCgYBTIevNPEIVrzncUyP56gu61z5YG/kDQ4+UpUgn6Ab2A1auFCa7mtNp +qa+LW6MpC6C5In6SSqotd5WbqxbLA6I7vH72psCeAgpkP12Sd31K/ikiwqj3EaEt +11ucC4/nb+WZQEn1sqPOzichZoyJyaGAjmCk40sdHk2ZQJZcvA78Gg== -----END RSA PRIVATE KEY----- diff --git a/test/config/integration/certs/upstreamcert.cfg b/test/config/integration/certs/upstreamcert.cfg index 4c6c2985ac48..4e2e760a861c 100644 --- a/test/config/integration/certs/upstreamcert.cfg +++ b/test/config/integration/certs/upstreamcert.cfg @@ -34,5 +34,5 @@ authorityKeyIdentifier = keyid:always [alt_names] DNS.1 = *.lyft.com -IP.1 = 0.0.0.0 -IP.2 = :: +IP.1 = 127.0.0.1 +IP.2 = ::1 \ No newline at end of file diff --git a/test/config/integration/certs/upstreamcert.pem b/test/config/integration/certs/upstreamcert.pem index 509397d6caa8..211fa9d9dff6 100644 --- a/test/config/integration/certs/upstreamcert.pem +++ b/test/config/integration/certs/upstreamcert.pem @@ -1,25 +1,25 @@ -----BEGIN CERTIFICATE----- -MIIEPjCCAyagAwIBAgIUS0ht/ypqxlVqt86GiCya6cw/jJwwDQYJKoZIhvcNAQEL +MIIEPjCCAyagAwIBAgIUEuy1WgSCzX6mojPirk7Th6uhNHowDQYJKoZIhvcNAQEL BQAwfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n -aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzA4MjE0 -NTU3WhcNMjEwNzA3MjE0NTU3WjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh +aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMjAwODA1MTkx +NjAyWhcNMjIwODA1MTkxNjAyWjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh bGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQx GTAXBgNVBAsMEEx5ZnQgRW5naW5lZXJpbmcxHTAbBgNVBAMMFFRlc3QgVXBzdHJl -YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+evHY2ld -y4iKlGEHtenqIK26QeFpU9t2iqiRjUz0+lZ+92haR+I+x1nL41SO71i2SaHp8L2Q -5cWkOS0zuYivsZSz/l9dHinAS+N50QsERo01moWOMyxfqSEbnZHMQS4OI/mf1dja -Rykp7zhCXie2BlUtmiBMW+YnvLmBm1z6icwg7ZBJ8mt2ChpeH6qBggzwQQms9wvK -/mcHR5HYHalLQdjhou3wwa6MB9bbeEoDd8I0tueRgnrq55mVJrm3yg1TSgUwkWCB -J3VUrvdk3olgGwHv4njAB+uNfUn3od7MuipyHL8GJQJHOcus63M/Ax/UVxu0BiDy -LfWW4MVO/5OX5QIDAQABo4GsMIGpMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg +YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtpiYA4/I +NuflkPe4L/GTslmngNQUCo8TzPXG0gt7uoxr4FeuVy7AaD28S2/hwhbl+bDtHTQY +mvBUwNMsYzpND2eQ3sSIumdeLzBEKP2mnnZ9gE/Zd2TIuZl686RpDq0B6ZdZSpCu +bqQmmPFLiRNH8JViJZMN5yqMt7T5oq+DnCYQZllqmpAwd6NnhKALrYmZ87oqc0zh +kf+5amP7zMYKkwQuRwcx4QPZkEp3+qhszolpAJ52dFGJ+pLuUVDg0Gf0cnxLjFKc +6vcTlj4tsymR4ci58MHRt4EdGdhShw0oaj67gRRfU4Vj61I2ZAVH07kL0mjO2TZT +EKrOEJJ7/dtxdwIDAQABo4GsMIGpMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg MB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAtBgNVHREEJjAkggoqLmx5 -ZnQuY29thwQAAAAAhxAAAAAAAAAAAAAAAAAAAAAAMB0GA1UdDgQWBBR3BvXiz3zi -p+/5cojIhCEz3nn39jAfBgNVHSMEGDAWgBTi0zC68szQHLsXt3ChIQpZC+yHbjAN -BgkqhkiG9w0BAQsFAAOCAQEAEB9RWuGIcRhZMM2AqXyOr4FOG63yfVg4fi3/WIcu -p7iVPhtdByefx4FQxg7913rdJyeQrI+hab0uPl/CjylwMVwWtBqRx4oKo8im59/4 -N7MRYZKJ44/fBSIGoM0pibSpDzfd7y6Drusp1mqi3CXGPXsVFIDQ66d7yoFt+t7h -nB2A565e/C1eXaS80XTHeJzfS5dJ6ssjgyszTGM5PdN9C335pDGfQV0CqGNAMZqo -tbBI1B0NgQ2KJJ787Wi3pexxi3haliMNrSKEAkLVDZ6R0a1PgpN/hBth3Nf2Oj+O -+pBNtkiA0fnkoKS6ps9Vgj+NB08OLeYNpfGFHa9xxFdPoA== +ZnQuY29thwR/AAABhxAAAAAAAAAAAAAAAAAAAAABMB0GA1UdDgQWBBQeoC5wxwX5 +k3ggIN/844/6jKx9czAfBgNVHSMEGDAWgBQ7Zh1TopMm7SY1RCOEO8L1G8IAZDAN +BgkqhkiG9w0BAQsFAAOCAQEA18wEg8LnPm99cIouFUFMAO+BpiY2KVa9Bu6x07m9 +quNFv7/4mLt87sk/umD3LH/tDjqW0D84vhG9a+0yDq7ZrD/P5eK3R+yBINwhe4/x +obJlThEsbcZF1FkMnq1rt53izukyQLLQwoVxidQl3HCg3hosWmpH1VBPgwoize6V +aAhKLW0n+JSfIE1d80nvZdYlHuCnS6UhLmAbTBCnwT0aGTfzT0Dd4KlYiY8vGZRu +tXOw4MzKtJcOL3t7Zpz2mhqN25dyiuyvKEhLXdx48aemwa2t6ISfFKsd0/glnNe/ +PFZMakzKv1G0xLGURjsInCZ0kePAmerfZN6CBZDo4laYEg== -----END CERTIFICATE----- diff --git a/test/config/integration/certs/upstreamcert_hash.h b/test/config/integration/certs/upstreamcert_hash.h index 4342078b9380..ec9a111a3155 100644 --- a/test/config/integration/certs/upstreamcert_hash.h +++ b/test/config/integration/certs/upstreamcert_hash.h @@ -1,3 +1,3 @@ // NOLINT(namespace-envoy) -constexpr char TEST_UPSTREAM_CERT_HASH[] = "57:0E:EF:74:60:9C:8E:3D:AA:EA:3F:3E:02:69:89:40:E6:00:" - "AD:CA:86:69:73:BA:9E:0B:01:A2:E2:F3:75:7F"; +constexpr char TEST_UPSTREAM_CERT_HASH[] = "9E:4B:E7:83:5B:DE:82:A6:E5:CE:24:8D:DB:91:C1:0C:20:0F:" + "25:3A:D4:4C:5E:13:AB:CD:53:00:93:85:F3:BE"; diff --git a/test/config/integration/certs/upstreamkey.pem b/test/config/integration/certs/upstreamkey.pem index 58945f2125ad..c1fc3c90cc7e 100644 --- a/test/config/integration/certs/upstreamkey.pem +++ b/test/config/integration/certs/upstreamkey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA+evHY2ldy4iKlGEHtenqIK26QeFpU9t2iqiRjUz0+lZ+92ha -R+I+x1nL41SO71i2SaHp8L2Q5cWkOS0zuYivsZSz/l9dHinAS+N50QsERo01moWO -MyxfqSEbnZHMQS4OI/mf1djaRykp7zhCXie2BlUtmiBMW+YnvLmBm1z6icwg7ZBJ -8mt2ChpeH6qBggzwQQms9wvK/mcHR5HYHalLQdjhou3wwa6MB9bbeEoDd8I0tueR -gnrq55mVJrm3yg1TSgUwkWCBJ3VUrvdk3olgGwHv4njAB+uNfUn3od7MuipyHL8G -JQJHOcus63M/Ax/UVxu0BiDyLfWW4MVO/5OX5QIDAQABAoIBAFqMy9w/8+TnntYt -5b5KdzLJ3x85jZD9hhCtDLd2d5gwOKZpX7SFy5ss9Mtz+qnLqZg6GunHtTUbC+pP -b1s8o/OiXii+4p0oIW0diShtZmothYtr8l6mKC6+OSQ5DBldl2//ZKL1g/ieeHwd -FSbKGpBm0jPymdf+Js2hJM1mvbuoy8ZxkdAtuYA/7tqQVG3/yFfB9Hm9JmGjU5iH -2m0qrZsch0pusKvw7zwoPshLcNvDeIt/i1gkoCNi5ZSxQ/Ow4dHaxSuQyggbzhn4 -j0SHpGtRhrOkccxKmc8EDxZynWYb5nCPAOSsb5SOtXMDrJdZUi0c9a2ZYLpcjz9m -RJH7a8ECgYEA/a0hrbUq43wNUuFeGs7W42u+R4L+bQ8aGF7MRkcF1CFfP9aYkebH -7DIt5Tz4ESCUKdM8SZLm1L+JsSQDq3T/8S+UrQpAvH/0FDbChGXe7mPTveMa7mTo -/l4gQ0BtqSknj68I0NGM30yuk1OdIRRFMFWEJZvRvb43JCDS2NRPnHUCgYEA/DXX -WNk9aABvW1IlCf0iufwHOWINNvaELHX1LhRhpgF/zinOjN+QBxibLyEQwD+x3zcH -SiN6xOt+KUuM+b7yeoPJhfMyCTENrMezOun89tTnpI6SYiPn6ugHeR8hQHPKe2X3 -T6sCY6KnzN29j8LICZJGRKeqKZUm006Lcoh1X7ECgYEAnRzztPB2Bbq5TdHDRPtC -YExE52mcRtOJp/perlAirgWVRqaUjBjRTdquTkJ6qbDx0w2/Uxom2TFgCFRz6Wdn -dWuwu5OUEKt28mYQB4xIjIFLjVnxPiFFpPWLKdvnj1Or6vPPk/WVOF/358trkCdL -yunMFLbzKn97C2dA74ZfYFkCgYEAvslb5fIv6YSquEIjkrLSmi50qIvrwzAoPBnf -JsR0OcfYjnRBs39KzJNokPZKXaPRQjG2afb84Anknghw1FwFwXf/8jxOFXXuCk3m -3yIyIeZcdLcFNQhEYAa14IIT/VWaTk6MDtAmNojMtsTmqOGHwPXOAhFzP5F8lUxN -YI6pe4ECgYA/Q3cX3R2P0WZq19+0IASRzuxSeIuT/Pw51/1qnESkFw5HvQ9HFSmT -J2lvWI0N4oyCBEuynVPFneR2lK2FN7ZOIVlQMFNQ6nJDFwvTQr4cXHn0eURTKKf1 -frP8QXXeP9rdsoo9veCciJpHZz22vpVE7FZlC0WDTOTl1kltO2z78A== +MIIEpAIBAAKCAQEAtpiYA4/INuflkPe4L/GTslmngNQUCo8TzPXG0gt7uoxr4Feu +Vy7AaD28S2/hwhbl+bDtHTQYmvBUwNMsYzpND2eQ3sSIumdeLzBEKP2mnnZ9gE/Z +d2TIuZl686RpDq0B6ZdZSpCubqQmmPFLiRNH8JViJZMN5yqMt7T5oq+DnCYQZllq +mpAwd6NnhKALrYmZ87oqc0zhkf+5amP7zMYKkwQuRwcx4QPZkEp3+qhszolpAJ52 +dFGJ+pLuUVDg0Gf0cnxLjFKc6vcTlj4tsymR4ci58MHRt4EdGdhShw0oaj67gRRf +U4Vj61I2ZAVH07kL0mjO2TZTEKrOEJJ7/dtxdwIDAQABAoIBACz6E1+1N/0GTA7U +ZgMxP09MNC1QkAs1yQvQcoPknjqKQjxFfMUu1+gVZN80FOjpGQbTJOTvoyvvDQFe +Qu3CO58SxKWKxZ8cvR9khTWPnU4lI67KfGejZKoK+zUuh049IV53kGAEmWLZfkRo +E1IVdL/3G/DjcyZA3d6WbnM7RnDcqORPnig4lq5HxN76eBdssbxtrAi3Sjy3ChMy +BLInnryF4UtaT5xqR26YjgtFmYkunrgXTe1i/ewQgBBkSPXcNr7or69hCCv0SG9e +vRsv1r+Uug3/iRZDjEhKBmXWNAZJ/IsDF37ywiyeBdUY+klDX+mWz+0BB0us8b4u +LxoZQTECgYEA2Gu9EVC0RMrQ9FF5AgKKJWmZKkOn346RkPrtbl5lbuUgnVdBXJjr +wfMZVTD/8E/tMN4EMSGmC9bxCpRRzhrphrm7SHGD6b9O30DH9q0TV0r0A8IG/bMO +xJLYjrYVxtEE+KckzvyvfIefbDG7wYkI3u+ObmjBg9t6jcErKlI/PdkCgYEA1/1E +T+cpR16iOPz1mz+f/GU4YmPkdSDj/PrjMv0c1OTDvdPiZPpLkhLUKiICnKSKbYjX +Ko8fdZc3cmakgD1dXtAfR7Tf/hXQIR5+iHD48I5e9DVlkqMNDObfj82ohTFKVe/P +ZSwgDiAPTMFxWr26u/GzY5D3adCQYJyKE2wTh88CgYEAu7vpzGVnmu0ciXNLNvUh +BQcvODxsGT9BArTI1Z7I+oOD4TjZmAuHJz1L0lypB7stk+BjXoND2K1hdr3moJUz +0gy3a0YdGd07++nkDBVi26xHNCNRkS2MN/TyKgnFpiuW1mOXSH5lc+7p2h7iMiY/ +LbQ8p4Xzp//xtZnFafbiqTECgYEAwDN5KZ1r5z24H/xCVv+cT46HSU7ZCr3VA9cC +fOouUOitouu9J9xviTJGKKQRLPFi2awOxKmN9ic1SRE7y35P60JKw5WaSdGBXydy +s9nMPMyEhM5Lb9y2jUeZo68ACl5dZvG63a4RbGBtHQF67KOvWvXvi2eCM2BMShyi +5jujeZMCgYAjewq1hVqL1FOD8sIFpmndsH3+Dfc7BJ/erqGOX9bQYGvJO4nCe+7K +4o8qFQf4jwdxu0iNxYJIMdn+l4/pz2e7GUFHjgMduUclf27Qj1p+8EyYqp6cmkzM +8mcwRkYo3aM70EmUu0Xxi3d5O5F1bIJ5MkgXaX/zSF2N02B3jXroxQ== -----END RSA PRIVATE KEY----- diff --git a/test/config/integration/certs/upstreamlocalhostcert.pem b/test/config/integration/certs/upstreamlocalhostcert.pem index 169d1c63e568..33b597dfc8e4 100644 --- a/test/config/integration/certs/upstreamlocalhostcert.pem +++ b/test/config/integration/certs/upstreamlocalhostcert.pem @@ -1,25 +1,25 @@ -----BEGIN CERTIFICATE----- -MIIEPTCCAyWgAwIBAgIUfoTig3pqtlASJyyhMZ2/x0Hg33UwDQYJKoZIhvcNAQEL +MIIEPTCCAyWgAwIBAgIUEuy1WgSCzX6mojPirk7Th6uhNHswDQYJKoZIhvcNAQEL BQAwfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n -aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzEyMjI0 -MzQ1WhcNMjEwNzExMjI0MzQ1WjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh +aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMjAwODA1MTkx +NjAzWhcNMjIwODA1MTkxNjAzWjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh bGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQx GTAXBgNVBAsMEEx5ZnQgRW5naW5lZXJpbmcxHTAbBgNVBAMMFFRlc3QgVXBzdHJl -YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApWnYvUff -dh+TcLEYxQiw+ZUaGfBedmVmaxOHAbsWwBcMcwt3ITAjRPLPFEUt/DUxgmXO80zo -6YOc9uUIUGU0vqIFTQP3JfS9kMevrvQIkZbsO2rtNMYQ+F7HOmGUS3RiKdNNbHnX -NKKPsHe/UFiFCBwxVCT+NSGI3yHZFUSFlvH9BEO+a1lx4pp2R7UJTLdBVaGx42t1 -pTTR2E2S6E1tKXhtS/qN4+X+dDaUFfi7mz+QiNFsYKu5QgO9f8ewfdOPj9MImZvG -4l37/eNtNjzwTMx8Ph4moTSo4yH9ZBGHffPDm4ljPXpLUfiVyIvzR6ygG468ODcP -umQjprHM3TTxlQIDAQABo4GrMIGoMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg +YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAut0zAnO+ +0q/IyxWG/LVbjh5gI2C9ZqNpFD+PK/bwPahg5JdtLnOWPwL+3BqjRcJ7/viocncL +eF3bhJ2hgF7gZGqd07juA3O9Qo+UUUgFvSsEGg2Y5c5gmD3rxdatxv/TnCmWn2pf +5366GmMQXCz99QGFuwUvIAGdApTjh65V32BZlgIwI9ArRy2v4YyW4hEaqiMeMaqB +Q994UCqlbhL/d3eBz3pEG0UIMkUPndTN42cWK8dGhSSHlJQkdAOk6x+24xY/qvjc +OZOtoO0GQu+SBwV+S0Wt75CI2k/IKYzMX5G7QFEcE6RgkL8Ag2DSG4bl/6Zf3C3z +ufWZwMlhjzZoHwIDAQABo4GrMIGoMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg MB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAsBgNVHREEJTAjgglsb2Nh -bGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwHQYDVR0OBBYEFFhoq2tgE+IN -uLyrUa1YpgsOpkwTMB8GA1UdIwQYMBaAFOLTMLryzNAcuxe3cKEhClkL7IduMA0G -CSqGSIb3DQEBCwUAA4IBAQB6jYcrAtO+8rLWdBp2W2tj9lgTHE8+W9BzMONrFwVo -PNMPHKPl2XfqKbv64sW3Z9sIA7ThSUu0nAe8i3ddaL4xQvPnaOwTdSnhykyMg2Hp -dDVAT8nCQb7Q87cuiFKE7o87XaQeOS3hgKeQ4uKexA48MkKwNYTVRMb7iC64yZcg -DY5H0nxRISs6pIAD9SPvPOQv+KZ35/LDJy59Kvso1zPoM9e+CJcdHU3hKM3RBK5J -FOjDL8qFair7vyjO9DUPDgoAZJntCQYGC0ToYxTe8cQVJD+sW5gqqIBLRchMr3gL -ni7rAL+oUyufHIEu3upRId+FdVF/hQMCLF2xk54/KX0V +bGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwHQYDVR0OBBYEFKsGHS0dB1v/ +Q0YBMcKXV0BC95TFMB8GA1UdIwQYMBaAFDtmHVOikybtJjVEI4Q7wvUbwgBkMA0G +CSqGSIb3DQEBCwUAA4IBAQAet+A9/JpMOcTstI8AVxVtJ+DuaPSl9L+kUSTPRwee +F1YeHz3i7ibhdheZcWKWJUaVYmU2VTzYTT3iPxB36KvpHEcsxRAaCyND9a48JeIn +awV+2KUD6QhFPWrtcM/TIzDcrl9qT69nnraVv1haXh/t5xRA30+h5aA4t9PnHxWA +mDv6Hslqi/NLCN1KBIZxmUspLkZFJT3uS7mdcI3d0m5tQx7W7UZr/40sbdxRjK3Y +XnCJVa51ZAkoJMEiDY7/zPraZbW1QiIaGBRTGVJ/rzJDvwnMo117GIqiRKKw9fo3 +GntHAThLZeCkdfAcDnzKLkOU6Sp9QHXd2pAC7H1z3EjP -----END CERTIFICATE----- diff --git a/test/config/integration/certs/upstreamlocalhostcert_hash.h b/test/config/integration/certs/upstreamlocalhostcert_hash.h index 9f81e65fcd83..545ee399138d 100644 --- a/test/config/integration/certs/upstreamlocalhostcert_hash.h +++ b/test/config/integration/certs/upstreamlocalhostcert_hash.h @@ -1,4 +1,4 @@ // NOLINT(namespace-envoy) constexpr char TEST_UPSTREAMLOCALHOST_CERT_HASH[] = - "44:A1:C4:AD:71:B5:AE:A0:A2:24:63:DA:C8:FF:0C:FC:26:6E:4B:D7:08:DE:64:60:34:A0:72:42:6E:18:BA:" - "87"; + "8C:49:C0:6C:4C:02:39:29:EB:5B:37:C6:62:02:7A:81:77:22:35:DD:99:1D:62:78:4C:95:5B:38:36:6C:1F:" + "1E"; diff --git a/test/config/integration/certs/upstreamlocalhostkey.pem b/test/config/integration/certs/upstreamlocalhostkey.pem index 5331d810952c..30b9c0c46263 100644 --- a/test/config/integration/certs/upstreamlocalhostkey.pem +++ b/test/config/integration/certs/upstreamlocalhostkey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEApWnYvUffdh+TcLEYxQiw+ZUaGfBedmVmaxOHAbsWwBcMcwt3 -ITAjRPLPFEUt/DUxgmXO80zo6YOc9uUIUGU0vqIFTQP3JfS9kMevrvQIkZbsO2rt -NMYQ+F7HOmGUS3RiKdNNbHnXNKKPsHe/UFiFCBwxVCT+NSGI3yHZFUSFlvH9BEO+ -a1lx4pp2R7UJTLdBVaGx42t1pTTR2E2S6E1tKXhtS/qN4+X+dDaUFfi7mz+QiNFs -YKu5QgO9f8ewfdOPj9MImZvG4l37/eNtNjzwTMx8Ph4moTSo4yH9ZBGHffPDm4lj -PXpLUfiVyIvzR6ygG468ODcPumQjprHM3TTxlQIDAQABAoIBAH2pMnFg937qL/0N -XM7acm+4eLK561kwYST5GbgT5A2btOZ1EFRTGIgZmX1BrNSLqIfyRcyJYet8A7OA -fNdueypTNYmzeH8KNTSWrn1PgG7x45aj/X349g1pGxrb5GeKC8TQdGHzEa03zcb2 -wY0NIkrt9/9/durwBeXU9fB1NLNc++Gbrok7kvbDcU6jN8Fas1H86beqhjTohkcN -C0lXk+VLi6m/lBMMjMlezqvTevMdBAjp0LfjwKLMMQE2JhqiS3GxVXhn9N3v0PoP -wI3PuwiUmcIMSY5LvPWD0iy5e1yNFNxJRoTNnnpLnuUmymBSzEeZp4XPrtMefA84 -5Q5BOeECgYEA0yK96/qyTnYcvflqLYukz1YZT029haRR+Yd+TLegVgC+j34q1IuC -TDc2ypHbjwWzkzZYz/6VFMLDXLCPimVS9rpn8vaunZDCLH1xTJbFSTjHOcx83RMT -EBiNH07R33UfdRGJtL5xoLj1DI971wy6LR3yuCzu04VolCMu+PO3Zx0CgYEAyI/t -mF96dZhHB8DDkVe+MRVOuazfkq5JtoUlv74mtKAbx15Jk+iy7G37qHNLRHCGegYj -7PQDZhuGbrda3tce956d9SqpA6uZRqp/aTQBWTfHe6OanXvHPKG1q1hTmfSuWi7/ -Izjh1AGDIaPxtWgx5v7AU8mX4UifhlTvl9KwktkCgYEAug4rfv/0kN/UhDR+NJSS -L4OX2iKPmG0tL88OpVxLln4hbyGnbJVjxPYC+o9+A5LqpBeIPAIELb9TmSKd2z9e -1L1/TMPFLGScN8hzRyK1x8iZB34Dqm1cpxp7gdNbbqcviWJjDzujthZHG0J1xxQY -HBoAAfzWmN8/QQugIRHj1KECgYA1Uo7IxBm6yhGYbheQvNNEGXYkx2FpjgzrCdtP -by67NxYrm1XUjTmEwnj2ADEysPgP2TIT/YwpyYekR/tQ48DH9NPqKr1kzGqj7xCQ -19LD9aCDrquc0xvVcujp9UHE3Ni+AWCz7Jud0gkbGItav6kE0RYxMJfAvZ4sCMjq -hImNgQKBgBHs0Hhg+pie4VJfXh3sxb6Px+WV7UlTyNyTbDMRgq5RKxrisA4BpSax -CjvX/IpTdsP5pNCjAdRK3zs/XC10/wa4wHmk0cXHM9IAFgIncVOuHCkZQyzQ1JJA -cbG7OQFr/UiHJuafueERY2HukfPHfgIUVM3MsXXNR5zIypseBQLW +MIIEowIBAAKCAQEAut0zAnO+0q/IyxWG/LVbjh5gI2C9ZqNpFD+PK/bwPahg5Jdt +LnOWPwL+3BqjRcJ7/viocncLeF3bhJ2hgF7gZGqd07juA3O9Qo+UUUgFvSsEGg2Y +5c5gmD3rxdatxv/TnCmWn2pf5366GmMQXCz99QGFuwUvIAGdApTjh65V32BZlgIw +I9ArRy2v4YyW4hEaqiMeMaqBQ994UCqlbhL/d3eBz3pEG0UIMkUPndTN42cWK8dG +hSSHlJQkdAOk6x+24xY/qvjcOZOtoO0GQu+SBwV+S0Wt75CI2k/IKYzMX5G7QFEc +E6RgkL8Ag2DSG4bl/6Zf3C3zufWZwMlhjzZoHwIDAQABAoIBAFhHQd7psX+1PeX7 +YI8oWn10ijSMck336x9+u3OosGxgjI3Rn+nu/077ak2vY+0D6TJWZLXW2Ztes+Md +2PtdVyL5X2BzoDYPSp0UWZxgqx1oIgLw44fFjMq/jhAj0GsP1veSii77wR0LOH5Y +yJTTSJKjynrFAzNar8NVdXxW9wiUnE9pXzXFEKhWY9zntYOxxzpSXdbUtQBWMnCU +3gVkkKoJHSNqiDjLDHeNkmScPirJw+if8B+N5pgzhcoBzGdMGIEHxwvd7PtjyaxJ +kw+XPJj/JIsipRBqXwhcjnbuy0gnHMiYgYS2i70iO853VWJvDdHYXNsF/wT2VLB0 +3jfcidkCgYEA4z7Q98cv6JcjMdtCBl4NAecxNgV+TLgy7eekWQUlVq2CXT8Ky72H +qcNF1PDNEKzruNMtITomEBoOJbhMzMr0HOccP/DEU/nhMludR3a7MCL2TQw56Dhv +iI9Pzd0v8HwTggd1T8WP+ABwHRLHpJrMHnaKxWdYppm180azmRhaMAsCgYEA0oJO +gpyJ+1G2w54pZ4zk17I0ZHANYb2dJ6bM1w2qGONHCEPfRWeo3oPy7zdjtgs2rDc9 +eYtnuRrGCuzEJRI72+Qk/oxEDaqLUlEARjFEgzo17PijcGz1CLRHBZbG2+F9JBDF +SlTG0lpl4i99xN4JHv3ysYWvgXlWpdVuNW/I0L0CgYA66ITJRpRvygYwnXMPLYBX +tvP12hS0lKd3Lq5W+VOFlbMOsxH8YORzKJDIs6elI/5zSiMP0wAc+nQiaRVXnWEM +wQh8ttBeKI+tOzyZUvkRcG7C6GF2hnK7RtNcPXN49uEjuwU5KbC5jHuDveONEyfI +2df9dl3vyjb1mqViEYMHowKBgH7jrQ9t7H5hMxmXLL4OX6Lk+E/Sez5/XUuZb7/x +rKZz2U1SHDNp2JDIWJd5e9Ev0TTd12B8d3lMejP7o//0jcBuNR56zkqukmx8Bv5I +lFPFstu0xE/wXYNxp53m1NeVhClJMqMrlu0VMHS2y8jvTfAwgyoeuzwAOAqeLGBp +kVLBAoGBAIU50ngoEEkuBa9jZiwRTUEnkcs6w5rxIiU9CHDKPkoki3m6Ke5ACE6X ++iwv75MpIeMwNCVOsQNascRMgLfSFMJwnHm9rruCqi3iCGEhvvANrSbl80jPVaFq +wuElkQJ67t/Kqj2Nxq84dtP8SCdYtu582xS0NyIFBBJ9EPausuIx -----END RSA PRIVATE KEY----- diff --git a/test/extensions/grpc_credentials/aws_iam/BUILD b/test/extensions/grpc_credentials/aws_iam/BUILD index 0796f78a871c..4c9b6e7f22bf 100644 --- a/test/extensions/grpc_credentials/aws_iam/BUILD +++ b/test/extensions/grpc_credentials/aws_iam/BUILD @@ -13,7 +13,6 @@ envoy_cc_test( name = "aws_iam_grpc_credentials_test", srcs = envoy_select_google_grpc(["aws_iam_grpc_credentials_test.cc"]), data = ["//test/config/integration/certs"], - tags = ["fails_on_windows"], deps = [ "//source/extensions/grpc_credentials:well_known_names", "//source/extensions/grpc_credentials/aws_iam:config", diff --git a/test/extensions/grpc_credentials/file_based_metadata/BUILD b/test/extensions/grpc_credentials/file_based_metadata/BUILD index 53cff427b2fe..b258c5c082a4 100644 --- a/test/extensions/grpc_credentials/file_based_metadata/BUILD +++ b/test/extensions/grpc_credentials/file_based_metadata/BUILD @@ -13,7 +13,6 @@ envoy_cc_test( name = "file_based_metadata_grpc_credentials_test", srcs = ["file_based_metadata_grpc_credentials_test.cc"], data = ["//test/config/integration/certs"], - tags = ["fails_on_windows"], deps = [ "//source/extensions/grpc_credentials:well_known_names", "//source/extensions/grpc_credentials/file_based_metadata:config", diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index b20ff0318398..2c55ea739720 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -443,8 +443,8 @@ makeTcpListenSocket(const Network::Address::InstanceConstSharedPtr& address) { } static Network::SocketPtr makeTcpListenSocket(uint32_t port, Network::Address::IpVersion version) { - return makeTcpListenSocket( - Network::Utility::parseInternetAddress(Network::Test::getAnyAddressString(version), port)); + return makeTcpListenSocket(Network::Utility::parseInternetAddress( + Network::Test::getLoopbackAddressString(version), port)); } static Network::SocketPtr From 374dca7905fc048be74169a7655d0462606555ad Mon Sep 17 00:00:00 2001 From: Petr Pchelko Date: Tue, 11 Aug 2020 05:30:15 -0700 Subject: [PATCH 32/67] Implement host_rewrite_path option (#12440) This implements a host_rewrite_path option for rewriting the Host header based on path. See rational in the linked issue. Note: the regex is executed on the path with query/fragment stripped. This is analogues to what regex_rewrite option is doing. Risk Level: Low Testing: added unit tests Docs Changes: document the new option in proto file Release Notes: added to current.rst Fixes #12430 Signed-off-by: Petr Pchelko --- .../config/route/v3/route_components.proto | 19 +++++++++- .../route/v4alpha/route_components.proto | 19 +++++++++- docs/root/version_history/current.rst | 2 + .../config/route/v3/route_components.proto | 19 +++++++++- .../route/v4alpha/route_components.proto | 19 +++++++++- source/common/router/config_impl.cc | 13 +++++++ source/common/router/config_impl.h | 2 + test/common/router/config_impl_test.cc | 37 ++++++++++++++++--- 8 files changed, 121 insertions(+), 9 deletions(-) diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index c35e210691c5..3643d449272e 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -545,7 +545,7 @@ message CorsPolicy { core.v3.RuntimeFractionalPercent shadow_enabled = 10; } -// [#next-free-field: 35] +// [#next-free-field: 36] message RouteAction { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RouteAction"; @@ -890,6 +890,23 @@ message RouteAction { // must come from trusted source. string host_rewrite_header = 29 [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; + + // Indicates that during forwarding, the host header will be swapped with + // the result of the regex substitution executed on path value with query and fragment removed. + // This is useful for transitioning variable content between path segment and subdomain. + // + // For example with the following config: + // + // .. code-block:: yaml + // + // host_rewrite_path_regex: + // pattern: + // google_re2: {} + // regex: "^/(.+)/.+$" + // substitution: \1 + // + // Would rewrite the host header to `envoyproxy.io` given the path `/envoyproxy.io/some/path`. + type.matcher.v3.RegexMatchAndSubstitute host_rewrite_path_regex = 35; } // Specifies the upstream timeout for the route. If not specified, the default is 15s. This diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto index f921ea506d99..d57cdef06def 100644 --- a/api/envoy/config/route/v4alpha/route_components.proto +++ b/api/envoy/config/route/v4alpha/route_components.proto @@ -547,7 +547,7 @@ message CorsPolicy { core.v4alpha.RuntimeFractionalPercent shadow_enabled = 10; } -// [#next-free-field: 35] +// [#next-free-field: 36] message RouteAction { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RouteAction"; @@ -886,6 +886,23 @@ message RouteAction { // must come from trusted source. string host_rewrite_header = 29 [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; + + // Indicates that during forwarding, the host header will be swapped with + // the result of the regex substitution executed on path value with query and fragment removed. + // This is useful for transitioning variable content between path segment and subdomain. + // + // For example with the following config: + // + // .. code-block:: yaml + // + // host_rewrite_path_regex: + // pattern: + // google_re2: {} + // regex: "^/(.+)/.+$" + // substitution: \1 + // + // Would rewrite the host header to `envoyproxy.io` given the path `/envoyproxy.io/some/path`. + type.matcher.v4alpha.RegexMatchAndSubstitute host_rewrite_path_regex = 35; } // Specifies the upstream timeout for the route. If not specified, the default is 15s. This diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 4a6ac04a2576..ae34f7c3d119 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -64,6 +64,8 @@ New Features * router: added new :ref:`envoy-ratelimited` retry policy, which allows retrying envoy's own rate limited responses. +* router: added new :ref:`host_rewrite_path_regex ` + option, which allows rewriting Host header based on path. * signal: added support for calling fatal error handlers without envoy's signal handler, via FatalErrorHandler::callFatalErrorHandlers(). * stats: added optional histograms to :ref:`cluster stats ` that track headers and body sizes of requests and responses. diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index f79f399d2140..0b8fe7908603 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -557,7 +557,7 @@ message CorsPolicy { [deprecated = true, (validate.rules).repeated = {items {string {max_bytes: 1024}}}]; } -// [#next-free-field: 35] +// [#next-free-field: 36] message RouteAction { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RouteAction"; @@ -899,6 +899,23 @@ message RouteAction { // must come from trusted source. string host_rewrite_header = 29 [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; + + // Indicates that during forwarding, the host header will be swapped with + // the result of the regex substitution executed on path value with query and fragment removed. + // This is useful for transitioning variable content between path segment and subdomain. + // + // For example with the following config: + // + // .. code-block:: yaml + // + // host_rewrite_path_regex: + // pattern: + // google_re2: {} + // regex: "^/(.+)/.+$" + // substitution: \1 + // + // Would rewrite the host header to `envoyproxy.io` given the path `/envoyproxy.io/some/path`. + type.matcher.v3.RegexMatchAndSubstitute host_rewrite_path_regex = 35; } // Specifies the upstream timeout for the route. If not specified, the default is 15s. This diff --git a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto index a8b6ae4459ce..0cad79e8aa77 100644 --- a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto @@ -547,7 +547,7 @@ message CorsPolicy { core.v4alpha.RuntimeFractionalPercent shadow_enabled = 10; } -// [#next-free-field: 35] +// [#next-free-field: 36] message RouteAction { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RouteAction"; @@ -895,6 +895,23 @@ message RouteAction { // must come from trusted source. string host_rewrite_header = 29 [(validate.rules).string = {well_known_regex: HTTP_HEADER_NAME strict: false}]; + + // Indicates that during forwarding, the host header will be swapped with + // the result of the regex substitution executed on path value with query and fragment removed. + // This is useful for transitioning variable content between path segment and subdomain. + // + // For example with the following config: + // + // .. code-block:: yaml + // + // host_rewrite_path_regex: + // pattern: + // google_re2: {} + // regex: "^/(.+)/.+$" + // substitution: \1 + // + // Would rewrite the host header to `envoyproxy.io` given the path `/envoyproxy.io/some/path`. + type.matcher.v4alpha.RegexMatchAndSubstitute host_rewrite_path_regex = 35; } // Specifies the upstream timeout for the route. If not specified, the default is 15s. This diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 73a2de20e6dd..0cf162262a2f 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -287,6 +287,14 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, ? absl::optional(Http::LowerCaseString( route.route().host_rewrite_header())) : absl::nullopt), + host_rewrite_path_regex_( + route.route().has_host_rewrite_path_regex() + ? Regex::Utility::parseRegex(route.route().host_rewrite_path_regex().pattern()) + : nullptr), + host_rewrite_path_regex_substitution_( + route.route().has_host_rewrite_path_regex() + ? route.route().host_rewrite_path_regex().substitution() + : ""), cluster_name_(route.route().cluster()), cluster_header_name_(route.route().cluster_header()), cluster_not_found_response_code_(ConfigUtility::parseClusterNotFoundResponseCode( route.route().cluster_not_found_response_code())), @@ -527,6 +535,11 @@ void RouteEntryImplBase::finalizeRequestHeaders(Http::RequestHeaderMap& headers, headers.setHost(header_value); } } + } else if (host_rewrite_path_regex_ != nullptr) { + const std::string path(headers.getPathValue()); + absl::string_view just_path(Http::PathUtil::removeQueryAndFragment(path)); + headers.setHost( + host_rewrite_path_regex_->replaceAll(just_path, host_rewrite_path_regex_substitution_)); } // Handle path rewrite diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index a32d19fbe742..4f4c0c8a7e1d 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -745,6 +745,8 @@ class RouteEntryImplBase : public RouteEntry, // to virtual host is currently safe. const bool auto_host_rewrite_; const absl::optional auto_host_rewrite_header_; + const Regex::CompiledMatcherPtr host_rewrite_path_regex_; + const std::string host_rewrite_path_regex_substitution_; const std::string cluster_name_; const Http::LowerCaseString cluster_header_name_; const Http::Code cluster_not_found_response_code_; diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 5d4ce7b56bc7..7585b4ec0524 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -705,12 +705,12 @@ TEST_F(RouteMatcherTest, TestRoutes) { prefix: "/host/rewrite/me" route: cluster: ats - host_rewrite: new_host + host_rewrite_literal: new_host - match: prefix: "/oldhost/rewrite/me" route: cluster: ats - host_rewrite: new_oldhost + host_rewrite_literal: new_oldhost - match: path: "/foo" case_sensitive: true @@ -728,7 +728,7 @@ TEST_F(RouteMatcherTest, TestRoutes) { case_sensitive: false route: cluster: ats - host_rewrite: new_host + host_rewrite_literal: new_host - match: path: "/FOOD" case_sensitive: false @@ -749,12 +749,21 @@ TEST_F(RouteMatcherTest, TestRoutes) { value: rewrote route: cluster: ats - auto_host_rewrite_header: x-rewrite-host + host_rewrite_header: x-rewrite-host - match: path: "/do-not-rewrite-host-with-header-value" route: cluster: ats - auto_host_rewrite_header: x-rewrite-host + host_rewrite_header: x-rewrite-host + - match: + path: "/rewrite-host-with-path-regex/envoyproxy.io" + route: + cluster: ats + host_rewrite_path_regex: + pattern: + google_re2: {} + regex: "^/.+/(.+)$" + substitution: \1 - match: prefix: "/" route: @@ -1022,6 +1031,24 @@ TEST_F(RouteMatcherTest, TestRoutes) { EXPECT_EQ("api.lyft.com", headers.get_(Http::Headers::get().Host)); } + // Rewrites host using path. + { + Http::TestRequestHeaderMapImpl headers = + genHeaders("api.lyft.com", "/rewrite-host-with-path-regex/envoyproxy.io", "GET"); + const RouteEntry* route = config.route(headers, 0)->routeEntry(); + route->finalizeRequestHeaders(headers, stream_info, true); + EXPECT_EQ("envoyproxy.io", headers.get_(Http::Headers::get().Host)); + } + + // Rewrites host using path, removes query parameters + { + Http::TestRequestHeaderMapImpl headers = genHeaders( + "api.lyft.com", "/rewrite-host-with-path-regex/envoyproxy.io?query=query", "GET"); + const RouteEntry* route = config.route(headers, 0)->routeEntry(); + route->finalizeRequestHeaders(headers, stream_info, true); + EXPECT_EQ("envoyproxy.io", headers.get_(Http::Headers::get().Host)); + } + // Case sensitive rewrite matching test. { Http::TestRequestHeaderMapImpl headers = From 49f6966c777ef1fa43f2d86a2ef590dd3928aa2e Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Tue, 11 Aug 2020 14:19:06 -0400 Subject: [PATCH 33/67] ocsp: add parsing utilities for ASN.1 OCSP responses (#12307) This is the first step in implementing OCSP stapling as described [here](https://docs.google.com/document/d/14Ji0Vq7Xbe9LXM6IsWQo8mgEnOB8Bo6TH75sSmFbCEE/edit#heading=h.v4mph477rsju), adding the capability on top of BoringSSL to parse and validate DER-encoded OCSP responses. It extracts enough to determine the revocation status of a single SSL certificate and the window of time for which the OCSP response can be considered valid. OCSP extensions are not currently processed. Signed-off-by: Daniel Goldstein --- .../transport_sockets/tls/ocsp/BUILD | 34 ++ .../tls/ocsp/asn1_utility.cc | 135 +++++++ .../transport_sockets/tls/ocsp/asn1_utility.h | 217 +++++++++++ .../transport_sockets/tls/ocsp/ocsp.cc | 303 +++++++++++++++ .../transport_sockets/tls/ocsp/ocsp.h | 293 +++++++++++++++ .../transport_sockets/tls/ocsp/BUILD | 41 ++ .../tls/ocsp/asn1_utility_test.cc | 349 ++++++++++++++++++ .../tls/ocsp/gen_unittest_ocsp_data.sh | 156 ++++++++ .../transport_sockets/tls/ocsp/ocsp_test.cc | 277 ++++++++++++++ tools/spelling/spelling_dictionary.txt | 3 + 10 files changed, 1808 insertions(+) create mode 100644 source/extensions/transport_sockets/tls/ocsp/BUILD create mode 100644 source/extensions/transport_sockets/tls/ocsp/asn1_utility.cc create mode 100644 source/extensions/transport_sockets/tls/ocsp/asn1_utility.h create mode 100644 source/extensions/transport_sockets/tls/ocsp/ocsp.cc create mode 100644 source/extensions/transport_sockets/tls/ocsp/ocsp.h create mode 100644 test/extensions/transport_sockets/tls/ocsp/BUILD create mode 100644 test/extensions/transport_sockets/tls/ocsp/asn1_utility_test.cc create mode 100755 test/extensions/transport_sockets/tls/ocsp/gen_unittest_ocsp_data.sh create mode 100644 test/extensions/transport_sockets/tls/ocsp/ocsp_test.cc diff --git a/source/extensions/transport_sockets/tls/ocsp/BUILD b/source/extensions/transport_sockets/tls/ocsp/BUILD new file mode 100644 index 000000000000..e2995afd45cd --- /dev/null +++ b/source/extensions/transport_sockets/tls/ocsp/BUILD @@ -0,0 +1,34 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_extension_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_library( + name = "ocsp_lib", + srcs = ["ocsp.cc"], + hdrs = ["ocsp.h"], + repository = "", + deps = [ + ":asn1_utility_lib", + "//include/envoy/common:time_interface", + "//include/envoy/ssl:context_config_interface", + "//source/extensions/transport_sockets/tls:utility_lib", + ], +) + +envoy_cc_library( + name = "asn1_utility_lib", + srcs = ["asn1_utility.cc"], + hdrs = ["asn1_utility.h"], + repository = "", + deps = [ + "//include/envoy/common:time_interface", + "//include/envoy/ssl:context_config_interface", + "//source/common/common:c_smart_ptr_lib", + ], +) diff --git a/source/extensions/transport_sockets/tls/ocsp/asn1_utility.cc b/source/extensions/transport_sockets/tls/ocsp/asn1_utility.cc new file mode 100644 index 000000000000..82cf430a7fd2 --- /dev/null +++ b/source/extensions/transport_sockets/tls/ocsp/asn1_utility.cc @@ -0,0 +1,135 @@ +#include "extensions/transport_sockets/tls/ocsp/asn1_utility.h" + +#include "common/common/c_smart_ptr.h" + +#include "absl/strings/ascii.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { +namespace Ocsp { + +namespace { +// A type adapter since OPENSSL_free accepts void*. +void freeOpensslString(char* str) { OPENSSL_free(str); } + +// `ASN1_INTEGER` is a type alias for `ASN1_STRING`. +// This static_cast is intentional to avoid the +// c-style cast performed in `M_ASN1_INTEGER_free`. +void freeAsn1Integer(ASN1_INTEGER* integer) { + ASN1_STRING_free(static_cast(integer)); +} +} // namespace + +absl::string_view Asn1Utility::cbsToString(CBS& cbs) { + auto str_head = reinterpret_cast(CBS_data(&cbs)); + return {str_head, CBS_len(&cbs)}; +} + +ParsingResult> Asn1Utility::getOptional(CBS& cbs, unsigned tag) { + int is_present; + CBS data; + if (!CBS_get_optional_asn1(&cbs, &data, &is_present, tag)) { + return "Failed to parse ASN.1 element tag"; + } + + return is_present ? absl::optional(data) : absl::nullopt; +} + +ParsingResult Asn1Utility::parseOid(CBS& cbs) { + CBS oid; + if (!CBS_get_asn1(&cbs, &oid, CBS_ASN1_OBJECT)) { + return absl::string_view("Input is not a well-formed ASN.1 OBJECT"); + } + CSmartPtr oid_text(CBS_asn1_oid_to_text(&oid)); + if (oid_text == nullptr) { + return absl::string_view("Failed to parse oid"); + } + + std::string oid_text_str(oid_text.get()); + return oid_text_str; +} + +ParsingResult Asn1Utility::parseGeneralizedTime(CBS& cbs) { + CBS elem; + if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_GENERALIZEDTIME)) { + return "Input is not a well-formed ASN.1 GENERALIZEDTIME"; + } + + auto time_str = cbsToString(elem); + // OCSP follows the RFC 5280 enforcement that `GENERALIZEDTIME` + // fields MUST be in UTC, so must be suffixed with a Z character. + // Local time or time differential, though a part of the `ASN.1` + // `GENERALIZEDTIME` spec, are not supported. + // Reference: https://tools.ietf.org/html/rfc5280#section-4.1.2.5.2 + if (time_str.length() > 0 && absl::ascii_toupper(time_str.at(time_str.length() - 1)) != 'Z') { + return "GENERALIZEDTIME must be in UTC"; + } + + absl::Time time; + auto utc_time_str = time_str.substr(0, time_str.length() - 1); + std::string parse_error; + if (!absl::ParseTime(GENERALIZED_TIME_FORMAT, utc_time_str, &time, &parse_error)) { + return "Error parsing string of GENERALIZEDTIME format"; + } + return absl::ToChronoTime(time); +} + +// Performs the following conversions to go from bytestring to hex integer +// `CBS` -> `ASN1_INTEGER` -> `BIGNUM` -> String. +ParsingResult Asn1Utility::parseInteger(CBS& cbs) { + CBS num; + if (!CBS_get_asn1(&cbs, &num, CBS_ASN1_INTEGER)) { + return absl::string_view("Input is not a well-formed ASN.1 INTEGER"); + } + + auto head = CBS_data(&num); + CSmartPtr asn1_integer( + c2i_ASN1_INTEGER(nullptr, &head, CBS_len(&num))); + if (asn1_integer != nullptr) { + BIGNUM num_bn; + BN_init(&num_bn); + ASN1_INTEGER_to_BN(asn1_integer.get(), &num_bn); + + CSmartPtr char_hex_number(BN_bn2hex(&num_bn)); + BN_free(&num_bn); + if (char_hex_number != nullptr) { + std::string hex_number(char_hex_number.get()); + return hex_number; + } + } + + return absl::string_view("Failed to parse ASN.1 INTEGER"); +} + +ParsingResult> Asn1Utility::parseOctetString(CBS& cbs) { + CBS value; + if (!CBS_get_asn1(&cbs, &value, CBS_ASN1_OCTETSTRING)) { + return "Input is not a well-formed ASN.1 OCTETSTRING"; + } + + auto data = reinterpret_cast(CBS_data(&value)); + return std::vector{data, data + CBS_len(&value)}; +} + +ParsingResult Asn1Utility::skipOptional(CBS& cbs, unsigned tag) { + if (!CBS_get_optional_asn1(&cbs, nullptr, nullptr, tag)) { + return "Failed to parse ASN.1 element tag"; + } + return absl::monostate(); +} + +ParsingResult Asn1Utility::skip(CBS& cbs, unsigned tag) { + if (!CBS_get_asn1(&cbs, nullptr, tag)) { + return "Failed to parse ASN.1 element"; + } + + return absl::monostate(); +} + +} // namespace Ocsp +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/ocsp/asn1_utility.h b/source/extensions/transport_sockets/tls/ocsp/asn1_utility.h new file mode 100644 index 000000000000..05bea0c2d19b --- /dev/null +++ b/source/extensions/transport_sockets/tls/ocsp/asn1_utility.h @@ -0,0 +1,217 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/exception.h" +#include "envoy/common/time.h" + +#include "common/common/assert.h" + +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "openssl/bn.h" +#include "openssl/bytestring.h" +#include "openssl/ssl.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { +namespace Ocsp { + +constexpr absl::string_view GENERALIZED_TIME_FORMAT = "%E4Y%m%d%H%M%S"; + +/** + * The result of parsing an `ASN.1` structure or an `absl::string_view` error + * description. + */ +template using ParsingResult = absl::variant; + +/** + * Construct a `T` from the data contained in the CBS&. Functions + * of this type must advance the input CBS& over the element. + */ +template using Asn1ParsingFunc = std::function(CBS&)>; + +/** + * Utility functions for parsing DER-encoded `ASN.1` objects. + * This relies heavily on the 'openssl/bytestring' API which + * is BoringSSL's recommended interface for parsing DER-encoded + * `ASN.1` data when there is not an existing wrapper. + * This is not a complete library for `ASN.1` parsing and primarily + * serves as abstractions for the OCSP module, but can be + * extended and moved into a general utility to support parsing of + * additional `ASN.1` objects. + * + * Each function adheres to the invariant that given a reference + * to a crypto `bytestring` (CBS&), it will parse the specified + * `ASN.1` element and advance `cbs` over it. + * + * An exception is thrown if the `bytestring` is malformed or does + * not match the specified `ASN.1` object. The position + * of `cbs` is not reliable after an exception is thrown. + */ +class Asn1Utility { +public: + ~Asn1Utility() = default; + + /** + * Extracts the full contents of `cbs` as a string. + * + * @param `cbs` a CBS& that refers to the current document position + * @returns absl::string_view containing the contents of `cbs` + */ + static absl::string_view cbsToString(CBS& cbs); + + /** + * Parses all elements of an `ASN.1` SEQUENCE OF. `parse_element` must + * advance its input CBS& over the entire element. + * + * @param cbs a CBS& that refers to an `ASN.1` SEQUENCE OF object + * @param parse_element an `Asn1ParsingFunc` used to parse each element + * @returns ParsingResult> containing the parsed elements of the sequence + * or an error string if `cbs` does not point to a well-formed + * SEQUENCE OF object. + */ + template + static ParsingResult> parseSequenceOf(CBS& cbs, Asn1ParsingFunc parse_element); + + /** + * Checks if an explicitly tagged optional element of `tag` is present and + * if so parses its value with `parse_data`. If the element is not present, + * `cbs` is not advanced. + * + * @param cbs a CBS& that refers to the current document position + * @param parse_data an `Asn1ParsingFunc` used to parse the data if present + * @return ParsingResult> with a `T` if `cbs` is of the specified tag, + * nullopt if the element has a different tag, or an error string if parsing fails. + */ + template + static ParsingResult> parseOptional(CBS& cbs, Asn1ParsingFunc parse_data, + unsigned tag); + + /** + * Returns whether or not an element explicitly tagged with `tag` is present + * at `cbs`. If so, `cbs` is advanced over the optional and assigns + * `data` to the inner element, if `data` is not nullptr. + * If `cbs` does not contain `tag`, `cbs` remains at the same position. + * + * @param cbs a CBS& that refers to the current document position + * @param an unsigned explicit tag indicating an optional value + * + * @returns ParsingResult whether `cbs` points to an element tagged with `tag` or + * an error string if parsing fails. + */ + static ParsingResult> getOptional(CBS& cbs, unsigned tag); + + /** + * @param cbs a CBS& that refers to an `ASN.1` OBJECT IDENTIFIER element + * @returns ParsingResult the `OID` encoded in `cbs` or an error + * string if `cbs` does not point to a well-formed OBJECT IDENTIFIER + */ + static ParsingResult parseOid(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` `GENERALIZEDTIME` element + * @returns ParsingResult the UTC timestamp encoded in `cbs` + * or an error string if `cbs` does not point to a well-formed + * `GENERALIZEDTIME` + */ + static ParsingResult parseGeneralizedTime(CBS& cbs); + + /** + * Parses an `ASN.1` INTEGER type into its hex string representation. + * `ASN.1` INTEGER types are arbitrary precision. + * If you're SURE the integer fits into a fixed-size int, + * use `CBS_get_asn1_*` functions for the given integer type instead. + * + * @param cbs a CBS& that refers to an `ASN.1` INTEGER element + * @returns ParsingResult a hex representation of the integer + * or an error string if `cbs` does not point to a well-formed INTEGER + */ + static ParsingResult parseInteger(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` `OCTETSTRING` element + * @returns ParsingResult> the octets in `cbs` or + * an error string if `cbs` does not point to a well-formed `OCTETSTRING` + */ + static ParsingResult> parseOctetString(CBS& cbs); + + /** + * Advance `cbs` over an `ASN.1` value of the class `tag` if that + * value is present. Otherwise, `cbs` stays in the same position. + * + * @param cbs a CBS& that refers to the current document position + * @param tag the tag of the value to skip + * @returns `ParsingResult` a unit type denoting success + * or an error string if parsing fails. + */ + static ParsingResult skipOptional(CBS& cbs, unsigned tag); + + /** + * Advance `cbs` over an `ASN.1` value of the class `tag`. + * + * @param cbs a CBS& that refers to the current document position + * @param tag the tag of the value to skip + * @returns `ParsingResult` a unit type denoting success + * or an error string if parsing fails. + */ + static ParsingResult skip(CBS& cbs, unsigned tag); +}; + +template +ParsingResult> Asn1Utility::parseSequenceOf(CBS& cbs, + Asn1ParsingFunc parse_element) { + CBS seq_elem; + std::vector vec; + + // Initialize seq_elem to first element in sequence. + if (!CBS_get_asn1(&cbs, &seq_elem, CBS_ASN1_SEQUENCE)) { + return "Expected sequence of ASN.1 elements."; + } + + while (CBS_data(&seq_elem) < CBS_data(&cbs)) { + // parse_element MUST advance seq_elem + auto elem_res = parse_element(seq_elem); + if (absl::holds_alternative(elem_res)) { + vec.push_back(absl::get<0>(elem_res)); + } else { + return absl::get<1>(elem_res); + } + } + + RELEASE_ASSERT(CBS_data(&cbs) == CBS_data(&seq_elem), + "Sequence tag length must match actual length or element parsing would fail"); + + return vec; +} + +template +ParsingResult> Asn1Utility::parseOptional(CBS& cbs, Asn1ParsingFunc parse_data, + unsigned tag) { + auto maybe_data_res = getOptional(cbs, tag); + + if (absl::holds_alternative(maybe_data_res)) { + return absl::get(maybe_data_res); + } + + auto maybe_data = absl::get>(maybe_data_res); + if (maybe_data) { + auto res = parse_data(maybe_data.value()); + if (absl::holds_alternative(res)) { + return absl::get<0>(res); + } + return absl::get<1>(res); + } + + return absl::nullopt; +} + +} // namespace Ocsp +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/ocsp/ocsp.cc b/source/extensions/transport_sockets/tls/ocsp/ocsp.cc new file mode 100644 index 000000000000..11f7d0aa2c90 --- /dev/null +++ b/source/extensions/transport_sockets/tls/ocsp/ocsp.cc @@ -0,0 +1,303 @@ +#include "extensions/transport_sockets/tls/ocsp/ocsp.h" + +#include "common/common/utility.h" + +#include "extensions/transport_sockets/tls/ocsp/asn1_utility.h" +#include "extensions/transport_sockets/tls/utility.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { +namespace Ocsp { + +namespace CertUtility = Envoy::Extensions::TransportSockets::Tls::Utility; + +namespace { + +template T unwrap(ParsingResult res) { + if (absl::holds_alternative(res)) { + return absl::get<0>(res); + } + + throw EnvoyException(std::string(absl::get<1>(res))); +} + +unsigned parseTag(CBS& cbs) { + unsigned tag; + if (!CBS_get_any_asn1_element(&cbs, nullptr, &tag, nullptr)) { + throw EnvoyException("Failed to parse ASN.1 element tag"); + } + return tag; +} + +std::unique_ptr readDerEncodedOcspResponse(const std::vector& der) { + CBS cbs; + CBS_init(&cbs, der.data(), der.size()); + + auto resp = Asn1OcspUtility::parseOcspResponse(cbs); + if (CBS_len(&cbs) != 0) { + throw EnvoyException("Data contained more than a single OCSP response"); + } + + return resp; +} + +void skipResponderId(CBS& cbs) { + // ResponderID ::= CHOICE { + // byName [1] Name, + // byKey [2] KeyHash + // } + // + // KeyHash ::= OCTET STRING -- SHA-1 hash of responder's public key + // (excluding the tag and length fields) + + if (unwrap(Asn1Utility::getOptional(cbs, CBS_ASN1_CONSTRUCTED | CBS_ASN1_CONTEXT_SPECIFIC | 1)) || + unwrap(Asn1Utility::getOptional(cbs, CBS_ASN1_CONSTRUCTED | CBS_ASN1_CONTEXT_SPECIFIC | 2))) { + return; + } + + throw EnvoyException(absl::StrCat("Unknown choice for Responder ID: ", parseTag(cbs))); +} + +void skipCertStatus(CBS& cbs) { + // CertStatus ::= CHOICE { + // good [0] IMPLICIT NULL, + // revoked [1] IMPLICIT RevokedInfo, + // unknown [2] IMPLICIT UnknownInfo + // } + if (!(unwrap(Asn1Utility::getOptional(cbs, CBS_ASN1_CONTEXT_SPECIFIC | 0)) || + unwrap( + Asn1Utility::getOptional(cbs, CBS_ASN1_CONSTRUCTED | CBS_ASN1_CONTEXT_SPECIFIC | 1)) || + unwrap(Asn1Utility::getOptional(cbs, CBS_ASN1_CONTEXT_SPECIFIC | 2)))) { + throw EnvoyException(absl::StrCat("Unknown OcspCertStatus tag: ", parseTag(cbs))); + } +} + +} // namespace + +OcspResponse::OcspResponse(OcspResponseStatus status, ResponsePtr response) + : status_(status), response_(std::move(response)) {} + +BasicOcspResponse::BasicOcspResponse(ResponseData data) : data_(data) {} + +ResponseData::ResponseData(std::vector single_responses) + : single_responses_(std::move(single_responses)) {} + +SingleResponse::SingleResponse(CertId cert_id, Envoy::SystemTime this_update, + absl::optional next_update) + : cert_id_(cert_id), this_update_(this_update), next_update_(next_update) {} + +CertId::CertId(std::string serial_number) : serial_number_(serial_number) {} + +OcspResponseWrapper::OcspResponseWrapper(std::vector der_response, TimeSource& time_source) + : raw_bytes_(std::move(der_response)), response_(readDerEncodedOcspResponse(raw_bytes_)), + time_source_(time_source) { + + if (response_->response_ == nullptr) { + throw EnvoyException("OCSP response has no body"); + } + + // We only permit a 1:1 of certificate to response. + if (response_->response_->getNumCerts() != 1) { + throw EnvoyException("OCSP Response must be for one certificate only"); + } + + auto& this_update = response_->response_->getThisUpdate(); + if (time_source_.systemTime() < this_update) { + std::string time_format(GENERALIZED_TIME_FORMAT); + DateFormatter formatter(time_format); + ENVOY_LOG_MISC(warn, "OCSP Response thisUpdate field is set in the future: {}", + formatter.fromTime(this_update)); + } +} + +// We use just the serial number to uniquely identify a certificate. +// Though different issuers could produce certificates with the same serial +// number, this is check is to prevent operator error and a collision in this +// case is unlikely. +bool OcspResponseWrapper::matchesCertificate(X509& cert) { + std::string cert_serial_number = CertUtility::getSerialNumberFromCertificate(cert); + std::string resp_cert_serial_number = response_->response_->getCertSerialNumber(); + return resp_cert_serial_number == cert_serial_number; +} + +bool OcspResponseWrapper::isExpired() { + auto& next_update = response_->response_->getNextUpdate(); + return next_update == absl::nullopt || next_update < time_source_.systemTime(); +} + +std::unique_ptr Asn1OcspUtility::parseOcspResponse(CBS& cbs) { + // OCSPResponse ::= SEQUENCE { + // responseStatus OCSPResponseStatus, + // responseBytes [0] EXPLICIT ResponseBytes OPTIONAL + // } + + CBS elem; + if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { + throw EnvoyException("OCSP Response is not a well-formed ASN.1 SEQUENCE"); + } + + OcspResponseStatus status = Asn1OcspUtility::parseResponseStatus(elem); + auto maybe_bytes = + unwrap(Asn1Utility::getOptional(elem, CBS_ASN1_CONSTRUCTED | CBS_ASN1_CONTEXT_SPECIFIC | 0)); + ResponsePtr resp = nullptr; + if (maybe_bytes) { + resp = Asn1OcspUtility::parseResponseBytes(maybe_bytes.value()); + } + + return std::make_unique(status, std::move(resp)); +} + +OcspResponseStatus Asn1OcspUtility::parseResponseStatus(CBS& cbs) { + // OCSPResponseStatus ::= ENUMERATED { + // successful (0), -- Response has valid confirmations + // malformedRequest (1), -- Illegal confirmation request + // internalError (2), -- Internal error in issuer + // tryLater (3), -- Try again later + // -- (4) is not used + // sigRequired (5), -- Must sign the request + // unauthorized (6) -- Request unauthorized + // } + CBS status; + if (!CBS_get_asn1(&cbs, &status, CBS_ASN1_ENUMERATED)) { + throw EnvoyException("OCSP ResponseStatus is not a well-formed ASN.1 ENUMERATED"); + } + + auto status_ordinal = *CBS_data(&status); + switch (status_ordinal) { + case 0: + return OcspResponseStatus::Successful; + case 1: + return OcspResponseStatus::MalformedRequest; + case 2: + return OcspResponseStatus::InternalError; + case 3: + return OcspResponseStatus::TryLater; + case 5: + return OcspResponseStatus::SigRequired; + case 6: + return OcspResponseStatus::Unauthorized; + default: + throw EnvoyException(absl::StrCat("Unknown OCSP Response Status variant: ", status_ordinal)); + } +} + +ResponsePtr Asn1OcspUtility::parseResponseBytes(CBS& cbs) { + // ResponseBytes ::= SEQUENCE { + // responseType RESPONSE. + // &id ({ResponseSet}), + // response OCTET STRING (CONTAINING RESPONSE. + // &Type({ResponseSet}{@responseType})) + // } + CBS elem, response; + if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { + throw EnvoyException("OCSP ResponseBytes is not a well-formed SEQUENCE"); + } + + auto oid_str = unwrap(Asn1Utility::parseOid(elem)); + if (!CBS_get_asn1(&elem, &response, CBS_ASN1_OCTETSTRING)) { + throw EnvoyException("Expected ASN.1 OCTETSTRING for response"); + } + + if (oid_str == BasicOcspResponse::OID) { + return Asn1OcspUtility::parseBasicOcspResponse(response); + } + throw EnvoyException(absl::StrCat("Unknown OCSP Response type with OID: ", oid_str)); +} + +std::unique_ptr Asn1OcspUtility::parseBasicOcspResponse(CBS& cbs) { + // BasicOCSPResponse ::= SEQUENCE { + // tbsResponseData ResponseData, + // signatureAlgorithm AlgorithmIdentifier{SIGNATURE-ALGORITHM, + // {`sa-dsaWithSHA1` | `sa-rsaWithSHA1` | + // `sa-rsaWithMD5` | `sa-rsaWithMD2`, ...}}, + // signature BIT STRING, + // certs [0] EXPLICIT SEQUENCE OF Certificate OPTIONAL + // } + CBS elem; + if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { + throw EnvoyException("OCSP BasicOCSPResponse is not a wellf-formed ASN.1 SEQUENCE"); + } + auto response_data = Asn1OcspUtility::parseResponseData(elem); + // The `signatureAlgorithm` and `signature` are ignored because OCSP + // responses are expected to be delivered from a reliable source. + // Optional additional certs are ignored. + + return std::make_unique(response_data); +} + +ResponseData Asn1OcspUtility::parseResponseData(CBS& cbs) { + // ResponseData ::= SEQUENCE { + // version [0] EXPLICIT Version DEFAULT v1, + // responderID ResponderID, + // producedAt GeneralizedTime, + // responses SEQUENCE OF SingleResponse, + // responseExtensions [1] EXPLICIT Extensions OPTIONAL + // } + CBS elem; + if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { + throw EnvoyException("OCSP ResponseData is not a well-formed ASN.1 SEQUENCE"); + } + + unwrap(Asn1Utility::skipOptional(elem, 0)); + skipResponderId(elem); + unwrap(Asn1Utility::skip(elem, CBS_ASN1_GENERALIZEDTIME)); + auto responses = unwrap(Asn1Utility::parseSequenceOf( + elem, [](CBS& cbs) -> ParsingResult { + return ParsingResult(parseSingleResponse(cbs)); + })); + // Extensions currently ignored. + + return {std::move(responses)}; +} + +SingleResponse Asn1OcspUtility::parseSingleResponse(CBS& cbs) { + // SingleResponse ::= SEQUENCE { + // certID CertID, + // certStatus CertStatus, + // thisUpdate GeneralizedTime, + // nextUpdate [0] EXPLICIT GeneralizedTime OPTIONAL, + // singleExtensions [1] EXPLICIT Extensions OPTIONAL + // } + CBS elem; + if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { + throw EnvoyException("OCSP SingleResponse is not a well-formed ASN.1 SEQUENCE"); + } + + auto cert_id = Asn1OcspUtility::parseCertId(elem); + skipCertStatus(elem); + auto this_update = unwrap(Asn1Utility::parseGeneralizedTime(elem)); + auto next_update = unwrap(Asn1Utility::parseOptional( + elem, Asn1Utility::parseGeneralizedTime, + CBS_ASN1_CONSTRUCTED | CBS_ASN1_CONTEXT_SPECIFIC | 0)); + // Extensions currently ignored. + + return {cert_id, this_update, next_update}; +} + +CertId Asn1OcspUtility::parseCertId(CBS& cbs) { + // CertID ::= SEQUENCE { + // hashAlgorithm AlgorithmIdentifier, + // issuerNameHash OCTET STRING, -- Hash of issuer's `DN` + // issuerKeyHash OCTET STRING, -- Hash of issuer's public key + // serialNumber CertificateSerialNumber + // } + CBS elem; + if (!CBS_get_asn1(&cbs, &elem, CBS_ASN1_SEQUENCE)) { + throw EnvoyException("OCSP CertID is not a well-formed ASN.1 SEQUENCE"); + } + + unwrap(Asn1Utility::skip(elem, CBS_ASN1_SEQUENCE)); + unwrap(Asn1Utility::skip(elem, CBS_ASN1_OCTETSTRING)); + unwrap(Asn1Utility::skip(elem, CBS_ASN1_OCTETSTRING)); + auto serial_number = unwrap(Asn1Utility::parseInteger(elem)); + + return {serial_number}; +} + +} // namespace Ocsp +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/ocsp/ocsp.h b/source/extensions/transport_sockets/tls/ocsp/ocsp.h new file mode 100644 index 000000000000..35c274ee1579 --- /dev/null +++ b/source/extensions/transport_sockets/tls/ocsp/ocsp.h @@ -0,0 +1,293 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/exception.h" +#include "envoy/common/time.h" + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "openssl/bytestring.h" +#include "openssl/ssl.h" + +/** + * Data structures and functions for unmarshaling OCSP responses + * according to the RFC6960 B.2 spec. See: https://tools.ietf.org/html/rfc6960#appendix-B + * + * WARNING: This module is meant to validate that OCSP responses are well-formed + * and extract useful fields for OCSP stapling. This assumes that responses are + * provided from configs or another trusted source and does not perform + * checks necessary to verify responses coming from an upstream server. + */ + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { +namespace Ocsp { + +/** + * Reflection of the `ASN.1` OcspResponseStatus enumeration. + * The possible statuses that can accompany an OCSP response. + */ +enum class OcspResponseStatus { + // OCSPResponseStatus ::= ENUMERATED { + // successful (0), -- Response has valid confirmations + // malformedRequest (1), -- Illegal confirmation request + // internalError (2), -- Internal error in issuer + // tryLater (3), -- Try again later + // -- (4) is not used + // sigRequired (5), -- Must sign the request + // unauthorized (6) -- Request unauthorized + // } + Successful = 0, + MalformedRequest = 1, + InternalError = 2, + TryLater = 3, + SigRequired = 5, + Unauthorized = 6 +}; + +/** + * Partial reflection of the `ASN.1` CertId structure. + * Contains the information to identify an SSL Certificate. + * Serial numbers are guaranteed to be + * unique per issuer but not necessarily universally. + */ +struct CertId { + CertId(std::string serial_number); + + std::string serial_number_; +}; + +/** + * Partial reflection of the `ASN.1` SingleResponse structure. + * Contains information about the OCSP status of a single certificate. + * An OCSP request may request the status of multiple certificates and + * therefore responses may contain multiple SingleResponses. + * + * this_update_ and next_update_ reflect the validity period for this response. + * If next_update_ is not present, the OCSP responder always has new information + * available. In this case the response would be considered immediately expired + * and invalid for stapling. + */ +struct SingleResponse { + SingleResponse(CertId cert_id, Envoy::SystemTime this_update, + absl::optional next_update); + + const CertId cert_id_; + const Envoy::SystemTime this_update_; + const absl::optional next_update_; +}; + +/** + * Partial reflection of the `ASN.1` ResponseData structure. + * Contains an OCSP response for each certificate in a given request + * as well as the time at which the response was produced. + */ +struct ResponseData { + ResponseData(std::vector single_responses); + + const std::vector single_responses_; +}; + +/** + * An abstract type for OCSP response formats. Which variant of `Response` is + * used in an `OcspResponse` is indicated by the structure's `OID`. + * + * Envoy enforces that OCSP responses must be for a single certificate + * only. The methods on this class extract the relevant information for the + * single certificate contained in the response. + */ +class Response { +public: + virtual ~Response() = default; + + /** + * @return The number of certs reported on by this response. + */ + virtual size_t getNumCerts() PURE; + + /** + * @return The serial number of the certificate. + */ + virtual const std::string& getCertSerialNumber() PURE; + + /** + * @return The beginning of the validity window for this response. + */ + virtual const Envoy::SystemTime& getThisUpdate() PURE; + + /** + * The time at which this response is considered to expire. If + * `nullopt`, then there is assumed to always be more up-to-date + * information available and the response is always considered expired. + * + * @return The end of the validity window for this response. + */ + virtual const absl::optional& getNextUpdate() PURE; +}; + +using ResponsePtr = std::unique_ptr; + +/** + * Reflection of the `ASN.1` BasicOcspResponse structure. + * Contains the full data of an OCSP response. + * Envoy enforces that OCSP responses contain a response for only + * a single certificate. + * + * BasicOcspResponse is the only supported Response type in RFC 6960. + */ +class BasicOcspResponse : public Response { +public: + BasicOcspResponse(ResponseData data); + + // Response + size_t getNumCerts() override { return data_.single_responses_.size(); } + const std::string& getCertSerialNumber() override { + return data_.single_responses_[0].cert_id_.serial_number_; + } + const Envoy::SystemTime& getThisUpdate() override { + return data_.single_responses_[0].this_update_; + } + const absl::optional& getNextUpdate() override { + return data_.single_responses_[0].next_update_; + } + + // Identified as `id-pkix-ocsp-basic` in + // https://tools.ietf.org/html/rfc6960#appendix-B.2 + constexpr static absl::string_view OID = "1.3.6.1.5.5.7.48.1.1"; + +private: + const ResponseData data_; +}; + +/** + * Reflection of the `ASN.1` OcspResponse structure. + * This is the top-level data structure for OCSP responses. + */ +struct OcspResponse { + OcspResponse(OcspResponseStatus status, ResponsePtr response); + + OcspResponseStatus status_; + ResponsePtr response_; +}; + +/** + * A wrapper used to own and query an OCSP response in DER-encoded format. + */ +class OcspResponseWrapper { +public: + OcspResponseWrapper(std::vector der_response, TimeSource& time_source); + + /** + * @return std::vector& a reference to the underlying bytestring representation + * of the OCSP response + */ + const std::vector& rawBytes() { return raw_bytes_; } + + /** + * @return OcspResponseStatus whether the OCSP response was successfully created + * or a status indicating an error in the OCSP process + */ + OcspResponseStatus getResponseStatus() { return response_->status_; } + + /** + * @param cert a X509& SSL certificate + * @returns bool whether this OCSP response contains the revocation status of `cert` + */ + bool matchesCertificate(X509& cert); + + /** + * Determines whether the OCSP response can no longer be considered valid. + * This can be true if the nextUpdate field of the response has passed + * or is not present, indicating that there is always more updated information + * available. + * + * @returns bool if the OCSP response is expired. + */ + bool isExpired(); + +private: + const std::vector raw_bytes_; + const std::unique_ptr response_; + TimeSource& time_source_; +}; + +using OcspResponseWrapperPtr = std::unique_ptr; + +/** + * `ASN.1` DER-encoded parsing functions similar to `Asn1Utility` but specifically + * for structures related to OCSP. + * + * Each function must advance `cbs` across the element it refers to. + */ +class Asn1OcspUtility { +public: + /** + * @param `cbs` a CBS& that refers to an `ASN.1` OcspResponse element + * @returns std::unique_ptr the OCSP response encoded in `cbs` + * @throws Envoy::EnvoyException if `cbs` does not contain a well-formed OcspResponse + * element. + */ + static std::unique_ptr parseOcspResponse(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` OcspResponseStatus element + * @returns OcspResponseStatus the OCSP response encoded in `cbs` + * @throws Envoy::EnvoyException if `cbs` does not contain a well-formed + * OcspResponseStatus element. + */ + static OcspResponseStatus parseResponseStatus(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` Response element + * @returns Response containing the content of an OCSP response + * @throws Envoy::EnvoyException if `cbs` does not contain a well-formed + * structure that is a valid Response type. + */ + static ResponsePtr parseResponseBytes(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` BasicOcspResponse element + * @returns BasicOcspResponse containing the content of an OCSP response + * @throws Envoy::EnvoyException if `cbs` does not contain a well-formed + * BasicOcspResponse element. + */ + static std::unique_ptr parseBasicOcspResponse(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` ResponseData element + * @returns ResponseData containing the content of an OCSP response relating + * to certificate statuses. + * @throws Envoy::EnvoyException if `cbs` does not contain a well-formed + * ResponseData element. + */ + static ResponseData parseResponseData(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` SingleResponse element + * @returns SingleResponse containing the id and revocation status of + * a single certificate. + * @throws Envoy::EnvoyException if `cbs` does not contain a well-formed + * SingleResponse element. + */ + static SingleResponse parseSingleResponse(CBS& cbs); + + /** + * @param cbs a CBS& that refers to an `ASN.1` CertId element + * @returns CertId containing the information necessary to uniquely identify + * an SSL certificate. + * @throws Envoy::EnvoyException if `cbs` does not contain a well-formed + * CertId element. + */ + static CertId parseCertId(CBS& cbs); +}; + +} // namespace Ocsp +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/transport_sockets/tls/ocsp/BUILD b/test/extensions/transport_sockets/tls/ocsp/BUILD new file mode 100644 index 000000000000..9f42f8d2ad05 --- /dev/null +++ b/test/extensions/transport_sockets/tls/ocsp/BUILD @@ -0,0 +1,41 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "ocsp_test", + srcs = [ + "ocsp_test.cc", + ], + data = [ + "gen_unittest_ocsp_data.sh", + ], + external_deps = ["ssl"], + deps = [ + "//source/common/filesystem:filesystem_lib", + "//source/extensions/transport_sockets/tls:utility_lib", + "//source/extensions/transport_sockets/tls/ocsp:ocsp_lib", + "//test/extensions/transport_sockets/tls:ssl_test_utils", + "//test/test_common:environment_lib", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + ], +) + +envoy_cc_test( + name = "asn1_utility_test", + srcs = [ + "asn1_utility_test.cc", + ], + external_deps = ["ssl"], + deps = [ + "//source/extensions/transport_sockets/tls/ocsp:asn1_utility_lib", + "//test/extensions/transport_sockets/tls:ssl_test_utils", + ], +) diff --git a/test/extensions/transport_sockets/tls/ocsp/asn1_utility_test.cc b/test/extensions/transport_sockets/tls/ocsp/asn1_utility_test.cc new file mode 100644 index 000000000000..e3299c39bd22 --- /dev/null +++ b/test/extensions/transport_sockets/tls/ocsp/asn1_utility_test.cc @@ -0,0 +1,349 @@ +#include + +#include "extensions/transport_sockets/tls/ocsp/asn1_utility.h" + +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { +namespace Ocsp { + +namespace { + +class Asn1UtilityTest : public testing::Test { +public: + // DER encoding of a single TLV `ASN.1` element. + // returns a pointer to the underlying buffer and transfers + // ownership to the caller. + uint8_t* asn1Encode(CBS& cbs, std::string& value, unsigned tag) { + bssl::ScopedCBB cbb; + CBB child; + auto data_head = reinterpret_cast(value.c_str()); + + EXPECT_TRUE(CBB_init(cbb.get(), 0)); + EXPECT_TRUE(CBB_add_asn1(cbb.get(), &child, tag)); + EXPECT_TRUE(CBB_add_bytes(&child, data_head, value.size())); + + uint8_t* buf; + size_t buf_len; + EXPECT_TRUE(CBB_finish(cbb.get(), &buf, &buf_len)); + + CBS_init(&cbs, buf, buf_len); + return buf; + } + + template + void expectParseResultErrorOnWrongTag(std::function(CBS&)> parse) { + CBS cbs; + CBS_init(&cbs, asn1_true.data(), asn1_true.size()); + EXPECT_NO_THROW(absl::get<1>(parse(cbs))); + } + + const std::vector asn1_true = {0x1u, 1, 0xff}; + const std::vector asn1_empty_seq = {0x30, 0}; +}; + +TEST_F(Asn1UtilityTest, ParseMethodsWrongTagTest) { + expectParseResultErrorOnWrongTag>>([](CBS& cbs) { + return Asn1Utility::parseSequenceOf>(cbs, Asn1Utility::parseOctetString); + }); + expectParseResultErrorOnWrongTag(Asn1Utility::parseOid); + expectParseResultErrorOnWrongTag(Asn1Utility::parseGeneralizedTime); + expectParseResultErrorOnWrongTag(Asn1Utility::parseInteger); + expectParseResultErrorOnWrongTag>(Asn1Utility::parseOctetString); +} + +TEST_F(Asn1UtilityTest, ToStringTest) { + CBS cbs; + absl::string_view str = "test"; + CBS_init(&cbs, reinterpret_cast(str.data()), str.size()); + EXPECT_EQ(str, Asn1Utility::cbsToString(cbs)); +} + +TEST_F(Asn1UtilityTest, ParseSequenceOfEmptySequenceTest) { + CBS cbs; + CBS_init(&cbs, asn1_empty_seq.data(), asn1_empty_seq.size()); + + std::vector> vec; + auto actual = absl::get<0>( + Asn1Utility::parseSequenceOf>(cbs, Asn1Utility::parseOctetString)); + EXPECT_EQ(vec, actual); +} + +TEST_F(Asn1UtilityTest, ParseSequenceOfMultipleElementSequenceTest) { + std::vector octet_seq = { + // SEQUENCE OF 3 2-byte elements + 0x30, + 3 * (2 + 2), + // 1st OCTET STRING + 0x4u, + 2, + 0x1, + 0x2, + // 2nd OCTET STRING + 0x4u, + 2, + 0x3, + 0x4, + // 3rd OCTET STRING + 0x4u, + 2, + 0x5, + 0x6, + }; + CBS cbs; + CBS_init(&cbs, octet_seq.data(), octet_seq.size()); + + std::vector> vec = {{0x1, 0x2}, {0x3, 0x4}, {0x5, 0x6}}; + auto actual = absl::get<0>( + Asn1Utility::parseSequenceOf>(cbs, Asn1Utility::parseOctetString)); + EXPECT_EQ(vec, actual); +} + +TEST_F(Asn1UtilityTest, SequenceOfLengthMismatchErrorTest) { + std::vector malformed = { + // SEQUENCE OF length wrongfully 2 instead of 4 bytes + 0x30, + 3, + // 1st OCTET STRING + 0x4u, + 2, + 0x1, + 0x2, + }; + CBS cbs; + CBS_init(&cbs, malformed.data(), malformed.size()); + + EXPECT_EQ("Input is not a well-formed ASN.1 OCTETSTRING", + absl::get<1>(Asn1Utility::parseSequenceOf>( + cbs, Asn1Utility::parseOctetString))); +} + +TEST_F(Asn1UtilityTest, SequenceOfMixedTypeErrorTest) { + std::vector mixed_type = { + // SEQUENCE OF 1 OCTET STRING and 1 BOOLEAN + 0x30, + 7, + // OCTET STRING + 0x4u, + 2, + 0x1, + 0x2, + // BOOLEAN true + 0x1u, + 1, + 0xff, + }; + CBS cbs; + CBS_init(&cbs, mixed_type.data(), mixed_type.size()); + + EXPECT_EQ("Input is not a well-formed ASN.1 OCTETSTRING", + absl::get<1>(Asn1Utility::parseSequenceOf>( + cbs, Asn1Utility::parseOctetString))); +} + +TEST_F(Asn1UtilityTest, GetOptionalTest) { + CBS cbs; + CBS_init(&cbs, asn1_true.data(), asn1_true.size()); + + const uint8_t* start = CBS_data(&cbs); + EXPECT_EQ(absl::nullopt, absl::get<0>(Asn1Utility::getOptional(cbs, CBS_ASN1_INTEGER))); + EXPECT_EQ(start, CBS_data(&cbs)); + + CBS value = absl::get<0>(Asn1Utility::getOptional(cbs, CBS_ASN1_BOOLEAN)).value(); + EXPECT_EQ(0xff, *CBS_data(&value)); +} + +TEST_F(Asn1UtilityTest, GetOptionalMissingValueTest) { + std::vector missing_val_bool = {0x1u, 1}; + CBS cbs; + CBS_init(&cbs, missing_val_bool.data(), missing_val_bool.size()); + + auto res = Asn1Utility::getOptional(cbs, CBS_ASN1_BOOLEAN); + EXPECT_TRUE(absl::holds_alternative(res)); + EXPECT_EQ("Failed to parse ASN.1 element tag", absl::get<1>(res)); +} + +TEST_F(Asn1UtilityTest, ParseOptionalTest) { + std::vector nothing; + std::vector explicit_optional_true = {0, 3, 0x1u, 1, 0xff}; + + CBS cbs_true, cbs_explicit_optional_true, cbs_empty_seq, cbs_nothing; + CBS_init(&cbs_true, asn1_true.data(), asn1_true.size()); + CBS_init(&cbs_explicit_optional_true, explicit_optional_true.data(), + explicit_optional_true.size()); + CBS_init(&cbs_empty_seq, asn1_empty_seq.data(), asn1_empty_seq.size()); + CBS_init(&cbs_nothing, nothing.data(), nothing.size()); + + auto parseBool = [](CBS& cbs) -> bool { + int res; + CBS_get_asn1_bool(&cbs, &res); + return res; + }; + + absl::optional expected(true); + EXPECT_EQ(expected, absl::get<0>(Asn1Utility::parseOptional(cbs_explicit_optional_true, + parseBool, 0))); + EXPECT_EQ(absl::nullopt, absl::get<0>(Asn1Utility::parseOptional(cbs_empty_seq, parseBool, + CBS_ASN1_BOOLEAN))); + EXPECT_EQ(absl::nullopt, absl::get<0>(Asn1Utility::parseOptional(cbs_nothing, parseBool, + CBS_ASN1_BOOLEAN))); +} + +TEST_F(Asn1UtilityTest, ParseOidTest) { + std::string oid = "1.1.1.1.1.1.1"; + + bssl::ScopedCBB cbb; + CBB child; + ASSERT_TRUE(CBB_init(cbb.get(), 0)); + ASSERT_TRUE(CBB_add_asn1(cbb.get(), &child, CBS_ASN1_OBJECT)); + ASSERT_TRUE(CBB_add_asn1_oid_from_text(&child, oid.c_str(), oid.size())); + + uint8_t* buf; + size_t buf_len; + CBS cbs; + ASSERT_TRUE(CBB_finish(cbb.get(), &buf, &buf_len)); + CBS_init(&cbs, buf, buf_len); + bssl::UniquePtr scoped(buf); + + EXPECT_EQ(oid, absl::get<0>(Asn1Utility::parseOid(cbs))); +} + +TEST_F(Asn1UtilityTest, ParseGeneralizedTimeWrongFormatErrorTest) { + std::string invalid_time = ""; + CBS cbs; + bssl::UniquePtr scoped(asn1Encode(cbs, invalid_time, CBS_ASN1_GENERALIZEDTIME)); + Asn1Utility::parseGeneralizedTime(cbs); + EXPECT_EQ("Input is not a well-formed ASN.1 GENERALIZEDTIME", + absl::get(Asn1Utility::parseGeneralizedTime(cbs))); +} + +TEST_F(Asn1UtilityTest, ParseGeneralizedTimeTest) { + std::string time = "20070614185900z"; + std::string expected_time = "20070614185900"; + + CBS cbs; + bssl::UniquePtr scoped(asn1Encode(cbs, time, CBS_ASN1_GENERALIZEDTIME)); + absl::Time expected = TestUtility::parseTime(expected_time, "%E4Y%m%d%H%M%S"); + auto actual = absl::get(Asn1Utility::parseGeneralizedTime(cbs)); + + EXPECT_EQ(absl::ToChronoTime(expected), actual); +} + +TEST_F(Asn1UtilityTest, TestParseGeneralizedTimeRejectsNonUTCTime) { + std::string local_time = "20070601145918"; + CBS cbs; + bssl::UniquePtr scoped(asn1Encode(cbs, local_time, CBS_ASN1_GENERALIZEDTIME)); + + EXPECT_EQ("GENERALIZEDTIME must be in UTC", + absl::get(Asn1Utility::parseGeneralizedTime(cbs))); +} + +TEST_F(Asn1UtilityTest, TestParseGeneralizedTimeInvalidTime) { + std::string ymd = "20070601Z"; + CBS cbs; + bssl::UniquePtr scoped(asn1Encode(cbs, ymd, CBS_ASN1_GENERALIZEDTIME)); + + EXPECT_EQ("Error parsing string of GENERALIZEDTIME format", + absl::get<1>(Asn1Utility::parseGeneralizedTime(cbs))); +} + +// Taken from +// https://boringssl.googlesource.com/boringssl/+/master/crypto/bytestring/cbb.c#531 +// because boringssl_fips does not yet implement `CBB_add_asn1_int64` +void cbbAddAsn1Int64(CBB* cbb, int64_t value) { + if (value >= 0) { + ASSERT_TRUE(CBB_add_asn1_uint64(cbb, value)); + } + + union { + int64_t i; + uint8_t bytes[sizeof(int64_t)]; + } u; + u.i = value; + int start = 7; + // Skip leading sign-extension bytes unless they are necessary. + while (start > 0 && (u.bytes[start] == 0xff && (u.bytes[start - 1] & 0x80))) { + start--; + } + + CBB child; + ASSERT_TRUE(CBB_add_asn1(cbb, &child, CBS_ASN1_INTEGER)); + for (int i = start; i >= 0; i--) { + ASSERT_TRUE(CBB_add_u8(&child, u.bytes[i])); + } + CBB_flush(cbb); +} + +TEST_F(Asn1UtilityTest, ParseIntegerTest) { + std::vector> integers = { + {1, "01"}, + {10, "0a"}, + {1000000, "0f4240"}, + {-1, "-01"}, + }; + bssl::ScopedCBB cbb; + CBS cbs; + uint8_t* buf; + size_t buf_len; + for (auto const& int_and_hex : integers) { + ASSERT_TRUE(CBB_init(cbb.get(), 0)); + cbbAddAsn1Int64(cbb.get(), int_and_hex.first); + ASSERT_TRUE(CBB_finish(cbb.get(), &buf, &buf_len)); + + CBS_init(&cbs, buf, buf_len); + bssl::UniquePtr scoped_buf(buf); + + EXPECT_EQ(int_and_hex.second, absl::get<0>(Asn1Utility::parseInteger(cbs))); + cbb.Reset(); + } +} + +TEST_F(Asn1UtilityTest, ParseOctetStringTest) { + std::vector data = {0x1, 0x2, 0x3}; + std::string data_str(data.begin(), data.end()); + CBS cbs; + bssl::UniquePtr scoped(asn1Encode(cbs, data_str, CBS_ASN1_OCTETSTRING)); + + EXPECT_EQ(data, absl::get<0>(Asn1Utility::parseOctetString(cbs))); +} + +TEST_F(Asn1UtilityTest, SkipOptionalPresentAdvancesTest) { + CBS cbs; + CBS_init(&cbs, asn1_empty_seq.data(), asn1_empty_seq.size()); + + const uint8_t* start = CBS_data(&cbs); + EXPECT_NO_THROW(absl::get<0>(Asn1Utility::skipOptional(cbs, CBS_ASN1_SEQUENCE))); + EXPECT_EQ(start + 2, CBS_data(&cbs)); +} + +TEST_F(Asn1UtilityTest, SkipOptionalNotPresentDoesNotAdvanceTest) { + CBS cbs; + CBS_init(&cbs, asn1_empty_seq.data(), asn1_empty_seq.size()); + + const uint8_t* start = CBS_data(&cbs); + EXPECT_NO_THROW(absl::get<0>(Asn1Utility::skipOptional(cbs, CBS_ASN1_BOOLEAN))); + EXPECT_EQ(start, CBS_data(&cbs)); +} + +TEST_F(Asn1UtilityTest, SkipOptionalMalformedTagTest) { + std::vector malformed_seq = {0x30}; + CBS cbs; + CBS_init(&cbs, malformed_seq.data(), malformed_seq.size()); + + EXPECT_EQ("Failed to parse ASN.1 element tag", + absl::get<1>(Asn1Utility::skipOptional(cbs, CBS_ASN1_SEQUENCE))); +} + +} // namespace + +} // namespace Ocsp +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/transport_sockets/tls/ocsp/gen_unittest_ocsp_data.sh b/test/extensions/transport_sockets/tls/ocsp/gen_unittest_ocsp_data.sh new file mode 100755 index 000000000000..a2a8f8b3a4e6 --- /dev/null +++ b/test/extensions/transport_sockets/tls/ocsp/gen_unittest_ocsp_data.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Create test certificates and OCSP responses for them for unittests. + +set -e + +trap cleanup EXIT +cleanup() { + rm *_index* + rm *.csr + rm *.cnf + rm *_serial* +} + +[[ -z "${TEST_TMPDIR}" ]] && TEST_TMPDIR="$(cd $(dirname $0) && pwd)" + +TEST_OCSP_DIR="${TEST_TMPDIR}/ocsp_test_data" +mkdir -p "${TEST_OCSP_DIR}" + +cd $TEST_OCSP_DIR + +################################################## +# Make the configuration file +################################################## + +# $1= $2= +generate_config() { +(cat << EOF +[ req ] +default_bits = 2048 +distinguished_name = req_distinguished_name + +[ req_distinguished_name ] +countryName = US +countryName_default = US +stateOrProvinceName = California +stateOrProvinceName_default = California +localityName = San Francisco +localityName_default = San Francisco +organizationName = Lyft +organizationName_default = Lyft +organizationalUnitName = Lyft Engineering +organizationalUnitName_default = Lyft Engineering +commonName = $1 +commonName_default = $1 +commonName_max = 64 + +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = ${TEST_OCSP_DIR} +certs = ${TEST_OCSP_DIR} +new_certs_dir = ${TEST_OCSP_DIR} +serial = ${TEST_OCSP_DIR} +database = ${TEST_OCSP_DIR}/$2_index.txt +serial = ${TEST_OCSP_DIR}/$2_serial + +private_key = ${TEST_OCSP_DIR}/$2_key.pem +certificate = ${TEST_OCSP_DIR}/$2_cert.pem + +default_days = 375 +default_md = sha256 +preserve = no +policy = policy_default + +[ policy_default ] +countryName = optional +stateOrProvinceName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + + +[ v3_ca ] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer +basicConstraints = critical, CA:true +keyUsage = critical, digitalSignature, cRLSign, keyCertSign +EOF +) > $1.cnf +} + +# $1= $2=[issuer name] +generate_ca() { + if [[ "$2" != "" ]]; then local EXTRA_ARGS="-CA $2_cert.pem -CAkey $2_key.pem -CAcreateserial"; fi + openssl genrsa -out $1_key.pem 2048 + openssl req -new -key $1_key.pem -out $1_cert.csr \ + -config $1.cnf -batch -sha256 + openssl x509 -req \ + -in $1_cert.csr -signkey $1_key.pem -out $1_cert.pem \ + -extensions v3_ca -extfile $1.cnf $EXTRA_ARGS +} + +# $1= $2= +generate_x509_cert() { + openssl genrsa -out $1_key.pem 2048 + openssl req -new -key $1_key.pem -out $1_cert.csr -config $1.cnf -batch -sha256 + openssl ca -config $1.cnf -notext -batch -in $1_cert.csr -out $1_cert.pem +} + +# $1= $2= $3= $4=[extra args] +generate_ocsp_response() { + # Generate an OCSP request + openssl ocsp -CAfile $2_cert.pem -issuer $2_cert.pem \ + -cert $1_cert.pem -reqout $3_ocsp_req.der + + # Generate the OCSP response + openssl ocsp -CA $2_cert.pem \ + -rkey $2_key.pem -rsigner $2_cert.pem -index $2_index.txt \ + -reqin $3_ocsp_req.der -respout $3_ocsp_resp.der $4 +} + +# $1= $2= +revoke_certificate() { + openssl ca -revoke $1_cert.pem -keyfile $2_key.pem -cert $2_cert.pem -config $2.cnf +} + +# Set up the CA +touch ca_index.txt +echo "unique_subject = no" > ca_index.txt.attr +echo 1000 > ca_serial +generate_config ca ca +generate_ca ca + +# Set up an intermediate CA with a different database +touch intermediate_ca_index.txt +echo "unique_subject = no" > intermediate_ca_index.txt.attr +echo 1000 > intermediate_ca_serial +generate_config intermediate_ca intermediate_ca +generate_ca intermediate_ca ca + +# Generate valid cert and OCSP response +generate_config good ca +generate_x509_cert good ca +generate_ocsp_response good ca good "-ndays 7" + +# Generate OCSP response with the responder key hash instead of name +generate_ocsp_response good ca responder_key_hash -resp_key_id + +# Generate and revoke a cert and create OCSP response +generate_config revoked ca +generate_x509_cert revoked ca +revoke_certificate revoked ca +generate_ocsp_response revoked ca revoked + +# Create OCSP response for cert unknown to the CA +generate_ocsp_response good intermediate_ca unknown + +# Generate an OCSP request/response for multiple certs +openssl ocsp -CAfile ca_cert.pem -issuer ca_cert.pem \ + -cert good_cert.pem -cert revoked_cert.pem -reqout multiple_cert_ocsp_req.der +openssl ocsp -CA ca_cert.pem \ + -rkey ca_key.pem -rsigner ca_cert.pem -index ca_index.txt \ + -reqin multiple_cert_ocsp_req.der -respout multiple_cert_ocsp_resp.der diff --git a/test/extensions/transport_sockets/tls/ocsp/ocsp_test.cc b/test/extensions/transport_sockets/tls/ocsp/ocsp_test.cc new file mode 100644 index 000000000000..aebca52c04ef --- /dev/null +++ b/test/extensions/transport_sockets/tls/ocsp/ocsp_test.cc @@ -0,0 +1,277 @@ +#include "common/filesystem/filesystem_impl.h" + +#include "extensions/transport_sockets/tls/ocsp/ocsp.h" +#include "extensions/transport_sockets/tls/utility.h" + +#include "test/extensions/transport_sockets/tls/ssl_test_utility.h" +#include "test/test_common/environment.h" +#include "test/test_common/logging.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "openssl/x509v3.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { +namespace Ocsp { + +namespace { + +namespace CertUtility = Envoy::Extensions::TransportSockets::Tls::Utility; + +class OcspFullResponseParsingTest : public testing::Test { +public: + static void SetUpTestSuite() { // NOLINT(readability-identifier-naming) + TestEnvironment::exec({TestEnvironment::runfilesPath( + "test/extensions/transport_sockets/tls/ocsp/gen_unittest_ocsp_data.sh")}); + } + + std::string fullPath(std::string filename) { + return TestEnvironment::substitute("{{ test_tmpdir }}/ocsp_test_data/" + filename); + } + + std::vector readFile(std::string filename) { + auto str = TestEnvironment::readFileToStringForTest(fullPath(filename)); + return {str.begin(), str.end()}; + } + + void setup(std::string response_filename) { + auto der_response = readFile(response_filename); + response_ = std::make_unique(der_response, time_system_); + EXPECT_EQ(response_->rawBytes(), der_response); + } + + void expectSuccessful() { + EXPECT_EQ(OcspResponseStatus::Successful, response_->getResponseStatus()); + } + + void expectCertificateMatches(std::string cert_filename) { + auto cert_ = readCertFromFile(fullPath(cert_filename)); + EXPECT_TRUE(response_->matchesCertificate(*cert_)); + } + +protected: + Event::SimulatedTimeSystem time_system_; + OcspResponseWrapperPtr response_; +}; + +TEST_F(OcspFullResponseParsingTest, GoodCertTest) { + setup("good_ocsp_resp.der"); + expectSuccessful(); + expectCertificateMatches("good_cert.pem"); + + auto cert = readCertFromFile(fullPath("revoked_cert.pem")); + EXPECT_FALSE(response_->matchesCertificate(*cert)); + + // Contains nextUpdate that is in the future + EXPECT_FALSE(response_->isExpired()); +} + +TEST_F(OcspFullResponseParsingTest, RevokedCertTest) { + setup("revoked_ocsp_resp.der"); + expectSuccessful(); + expectCertificateMatches("revoked_cert.pem"); + EXPECT_TRUE(response_->isExpired()); +} + +TEST_F(OcspFullResponseParsingTest, UnknownCertTest) { + setup("unknown_ocsp_resp.der"); + expectSuccessful(); + expectCertificateMatches("good_cert.pem"); + EXPECT_TRUE(response_->isExpired()); +} + +TEST_F(OcspFullResponseParsingTest, ExpiredResponseTest) { + auto next_week = time_system_.systemTime() + std::chrono::hours(8 * 24); + time_system_.setSystemTime(next_week); + setup("good_ocsp_resp.der"); + // nextUpdate is present but in the past + EXPECT_TRUE(response_->isExpired()); +} + +TEST_F(OcspFullResponseParsingTest, ThisUpdateAfterNowTest) { + auto past_time = TestUtility::parseTime("2000 01 01", "%Y %m %d"); + time_system_.setSystemTime(absl::ToChronoTime(past_time)); + EXPECT_LOG_CONTAINS("warning", "OCSP Response thisUpdate field is set in the future", + setup("good_ocsp_resp.der")); +} + +TEST_F(OcspFullResponseParsingTest, ResponderIdKeyHashTest) { + setup("responder_key_hash_ocsp_resp.der"); + expectSuccessful(); + expectCertificateMatches("good_cert.pem"); + EXPECT_TRUE(response_->isExpired()); +} + +TEST_F(OcspFullResponseParsingTest, MultiCertResponseTest) { + auto resp_bytes = readFile("multiple_cert_ocsp_resp.der"); + EXPECT_THROW_WITH_MESSAGE(OcspResponseWrapper response(resp_bytes, time_system_), EnvoyException, + "OCSP Response must be for one certificate only"); +} + +TEST_F(OcspFullResponseParsingTest, NoResponseBodyTest) { + std::vector data = { + // SEQUENCE + 0x30, 3, + // OcspResponseStatus - InternalError + 0xau, 1, 2, + // no response bytes + }; + EXPECT_THROW_WITH_MESSAGE(OcspResponseWrapper response(data, time_system_), EnvoyException, + "OCSP response has no body"); +} + +TEST_F(OcspFullResponseParsingTest, OnlyOneResponseInByteStringTest) { + auto resp_bytes = readFile("good_ocsp_resp.der"); + auto resp2_bytes = readFile("revoked_ocsp_resp.der"); + resp_bytes.insert(resp_bytes.end(), resp2_bytes.begin(), resp2_bytes.end()); + + EXPECT_THROW_WITH_MESSAGE(OcspResponseWrapper response_wrapper(resp_bytes, time_system_), + EnvoyException, "Data contained more than a single OCSP response"); +} + +TEST_F(OcspFullResponseParsingTest, ParseOcspResponseWrongTagTest) { + auto resp_bytes = readFile("good_ocsp_resp.der"); + // Change the SEQUENCE tag to an `OCTETSTRING` tag + resp_bytes[0] = 0x4u; + EXPECT_THROW_WITH_MESSAGE(OcspResponseWrapper response_wrapper(resp_bytes, time_system_), + EnvoyException, "OCSP Response is not a well-formed ASN.1 SEQUENCE"); +} + +class Asn1OcspUtilityTest : public testing::Test { +public: + void expectResponseStatus(uint8_t code, OcspResponseStatus expected) { + std::vector asn1_enum = {0xau, 1, code}; + CBS cbs; + CBS_init(&cbs, asn1_enum.data(), asn1_enum.size()); + + EXPECT_EQ(expected, Asn1OcspUtility::parseResponseStatus(cbs)); + } + + void expectThrowOnWrongTag(std::function parse) { + CBS cbs; + CBS_init(&cbs, asn1_true.data(), asn1_true.size()); + EXPECT_THROW(parse(cbs), EnvoyException); + } + + const std::vector asn1_true = {0x1u, 1, 0xff}; +}; + +TEST_F(Asn1OcspUtilityTest, ParseResponseStatusTest) { + expectResponseStatus(0, OcspResponseStatus::Successful); + expectResponseStatus(1, OcspResponseStatus::MalformedRequest); + expectResponseStatus(2, OcspResponseStatus::InternalError); + expectResponseStatus(3, OcspResponseStatus::TryLater); + expectResponseStatus(5, OcspResponseStatus::SigRequired); + expectResponseStatus(6, OcspResponseStatus::Unauthorized); +} + +TEST_F(Asn1OcspUtilityTest, ParseMethodWrongTagTest) { + expectThrowOnWrongTag(Asn1OcspUtility::parseResponseBytes); + expectThrowOnWrongTag(Asn1OcspUtility::parseBasicOcspResponse); + expectThrowOnWrongTag(Asn1OcspUtility::parseResponseData); + expectThrowOnWrongTag(Asn1OcspUtility::parseSingleResponse); + expectThrowOnWrongTag(Asn1OcspUtility::parseCertId); + expectThrowOnWrongTag(Asn1OcspUtility::parseResponseStatus); +} + +TEST_F(Asn1OcspUtilityTest, ParseResponseDataBadResponderIdVariantTest) { + std::vector data = { + // SEQUENCE + 0x30, + 6, + // version + 0, + 1, + 0, + // Invalid Responder ID tag 3 + 3, + 1, + 0, + }; + CBS cbs; + CBS_init(&cbs, data.data(), data.size()); + EXPECT_THROW_WITH_MESSAGE(Asn1OcspUtility::parseResponseData(cbs), EnvoyException, + "Unknown choice for Responder ID: 3"); +} + +TEST_F(Asn1OcspUtilityTest, ParseOcspResponseBytesMissingTest) { + std::vector data = { + // SEQUENCE + 0x30, 3, + // OcspResponseStatus - InternalError + 0xau, 1, 2, + // no response bytes + }; + CBS cbs; + CBS_init(&cbs, data.data(), data.size()); + auto response = Asn1OcspUtility::parseOcspResponse(cbs); + EXPECT_EQ(response->status_, OcspResponseStatus::InternalError); + EXPECT_TRUE(response->response_ == nullptr); +} + +TEST_F(Asn1OcspUtilityTest, ParseResponseStatusUnknownVariantTest) { + std::vector bad_enum_variant = {0xau, 1, 4}; + CBS cbs; + CBS_init(&cbs, bad_enum_variant.data(), bad_enum_variant.size()); + EXPECT_THROW_WITH_MESSAGE(Asn1OcspUtility::parseResponseStatus(cbs), EnvoyException, + "Unknown OCSP Response Status variant: 4"); +} + +TEST_F(Asn1OcspUtilityTest, ParseResponseBytesNoOctetStringTest) { + std::string oid_str = "1.1.1.1.1.1.1"; + bssl::ScopedCBB cbb; + CBB seq, oid, obj; + uint8_t* buf; + size_t buf_len; + + ASSERT_TRUE(CBB_init(cbb.get(), 0)); + ASSERT_TRUE(CBB_add_asn1(cbb.get(), &seq, CBS_ASN1_SEQUENCE)); + ASSERT_TRUE(CBB_add_asn1(&seq, &oid, CBS_ASN1_OBJECT)); + ASSERT_TRUE(CBB_add_asn1_oid_from_text(&oid, oid_str.c_str(), oid_str.size())); + // Empty sequence instead of `OCTETSTRING` with the response + ASSERT_TRUE(CBB_add_asn1(&seq, &obj, CBS_ASN1_SEQUENCE)); + ASSERT_TRUE(CBB_finish(cbb.get(), &buf, &buf_len)); + + CBS cbs; + CBS_init(&cbs, buf, buf_len); + bssl::UniquePtr scoped(buf); + + EXPECT_THROW_WITH_MESSAGE(Asn1OcspUtility::parseResponseBytes(cbs), EnvoyException, + "Expected ASN.1 OCTETSTRING for response"); +} + +TEST_F(Asn1OcspUtilityTest, ParseResponseBytesUnknownResponseTypeTest) { + std::string oid_str = "1.1.1.1.1.1.1"; + bssl::ScopedCBB cbb; + CBB seq, oid, obj; + uint8_t* buf; + size_t buf_len; + + ASSERT_TRUE(CBB_init(cbb.get(), 0)); + ASSERT_TRUE(CBB_add_asn1(cbb.get(), &seq, CBS_ASN1_SEQUENCE)); + ASSERT_TRUE(CBB_add_asn1(&seq, &oid, CBS_ASN1_OBJECT)); + ASSERT_TRUE(CBB_add_asn1_oid_from_text(&oid, oid_str.c_str(), oid_str.size())); + ASSERT_TRUE(CBB_add_asn1(&seq, &obj, CBS_ASN1_OCTETSTRING)); + ASSERT_TRUE(CBB_add_bytes(&obj, reinterpret_cast("\x1\x2\x3"), 3)); + ASSERT_TRUE(CBB_finish(cbb.get(), &buf, &buf_len)); + + CBS cbs; + CBS_init(&cbs, buf, buf_len); + bssl::UniquePtr scoped(buf); + + EXPECT_THROW_WITH_MESSAGE(Asn1OcspUtility::parseResponseBytes(cbs), EnvoyException, + "Unknown OCSP Response type with OID: 1.1.1.1.1.1.1"); +} + +} // namespace + +} // namespace Ocsp +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 40f0ea30ffce..4ffedaa0d467 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -420,6 +420,7 @@ bazel behaviour benchmarked bidi +bignum bitfield bitset bitwise @@ -430,6 +431,7 @@ bool boolean booleans bools +boringssl borks broadcasted buf @@ -441,6 +443,7 @@ bulkstrings bursty bytecode bytestream +bytestring cacheable cacheability callee From 494583ca99e3b69360d3a7d436ed37672bd3e221 Mon Sep 17 00:00:00 2001 From: Yifan Yang Date: Tue, 11 Aug 2020 19:01:26 -0400 Subject: [PATCH 34/67] header: getting rid of exception-throwing behaviors in header files [include/ and source/common/] (#12502) This is a part of efforts to get rid of all the exception throwing semantics in envoy header files, as discussed here #12469. This PR deals with all instances that throw plain envoy exceptions in the described sub modules. Signed-off-by: Yifan Yang --- include/envoy/buffer/BUILD | 1 + include/envoy/buffer/buffer.h | 3 ++- include/envoy/registry/registry.h | 7 +++++-- include/envoy/stream_info/BUILD | 5 ++++- include/envoy/stream_info/filter_state.h | 5 +++-- source/common/common/utility.cc | 4 ++++ source/common/common/utility.h | 9 +++++++++ source/common/config/grpc_mux_impl.h | 3 ++- source/common/config/utility.h | 19 +++++++++++-------- source/common/event/BUILD | 1 + source/common/event/timer_impl.h | 3 ++- source/common/network/lc_trie.h | 12 +++++++----- 12 files changed, 51 insertions(+), 21 deletions(-) diff --git a/include/envoy/buffer/BUILD b/include/envoy/buffer/BUILD index 3f2880cb720e..d16a2fe36505 100644 --- a/include/envoy/buffer/BUILD +++ b/include/envoy/buffer/BUILD @@ -18,5 +18,6 @@ envoy_cc_library( "//include/envoy/api:os_sys_calls_interface", "//include/envoy/network:io_handle_interface", "//source/common/common:byte_order_lib", + "//source/common/common:utility_lib", ], ) diff --git a/include/envoy/buffer/buffer.h b/include/envoy/buffer/buffer.h index 6e4f52644e37..ab2d9af281ac 100644 --- a/include/envoy/buffer/buffer.h +++ b/include/envoy/buffer/buffer.h @@ -12,6 +12,7 @@ #include "envoy/network/io_handle.h" #include "common/common/byte_order.h" +#include "common/common/utility.h" #include "absl/container/inlined_vector.h" #include "absl/strings/string_view.h" @@ -286,7 +287,7 @@ class Instance { static_assert(Size <= sizeof(T), "requested size is bigger than integer being read"); if (length() < start + Size) { - throw EnvoyException("buffer underflow"); + ExceptionUtil::throwEnvoyException("buffer underflow"); } constexpr const auto displacement = Endianness == ByteOrder::BigEndian ? sizeof(T) - Size : 0; diff --git a/include/envoy/registry/registry.h b/include/envoy/registry/registry.h index 2b85df27c2e9..b52686036074 100644 --- a/include/envoy/registry/registry.h +++ b/include/envoy/registry/registry.h @@ -12,6 +12,7 @@ #include "common/common/assert.h" #include "common/common/fmt.h" #include "common/common/logger.h" +#include "common/common/utility.h" #include "common/config/api_type_oracle.h" #include "common/protobuf/utility.h" @@ -217,7 +218,8 @@ template class FactoryRegistry : public Logger::Loggable class FactoryRegistry : public Logger::Loggable const T& getDataReadOnly(absl::string_view data_name) const { const T* result = dynamic_cast(getDataReadOnlyGeneric(data_name)); if (!result) { - throw EnvoyException( + ExceptionUtil::throwEnvoyException( fmt::format("Data stored under {} cannot be coerced to specified type", data_name)); } return *result; @@ -111,7 +112,7 @@ class FilterState { template T& getDataMutable(absl::string_view data_name) { T* result = dynamic_cast(getDataMutableGeneric(data_name)); if (!result) { - throw EnvoyException( + ExceptionUtil::throwEnvoyException( fmt::format("Data stored under {} cannot be coerced to specified type", data_name)); } return *result; diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 1d7a5005129a..efdbb3ee29d5 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -572,4 +572,8 @@ InlineString::InlineString(const char* str, size_t size) : size_(size) { memcpy(data_, str, size); } +void ExceptionUtil::throwEnvoyException(const std::string& message) { + throw EnvoyException(message); +} + } // namespace Envoy diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 0101fd3d9fd9..f91d74832b60 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -619,6 +619,15 @@ template struct TrieLookupTable { TrieEntry root_; }; +/** + * A global utility class to take care of all the exception throwing behaviors in header files. + * Its functions simply forward the throwing into .cc file. + */ +class ExceptionUtil { +public: + [[noreturn]] static void throwEnvoyException(const std::string& message); +}; + // Mix-in class for allocating classes with variable-sized inlined storage. // // Use this class by inheriting from it, ensuring that: diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index d735bc12c1cf..232559ad2167 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -16,6 +16,7 @@ #include "common/common/cleanup.h" #include "common/common/logger.h" +#include "common/common/utility.h" #include "common/config/api_version.h" #include "common/config/grpc_stream.h" #include "common/config/utility.h" @@ -159,7 +160,7 @@ class NullGrpcMuxImpl : public GrpcMux, GrpcMuxWatchPtr addWatch(const std::string&, const std::set&, SubscriptionCallbacks&, OpaqueResourceDecoder&) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); + ExceptionUtil::throwEnvoyException("ADS must be configured to support an ADS config source"); } void onWriteable() override {} diff --git a/source/common/config/utility.h b/source/common/config/utility.h index d19026386c53..0e5f42c6c847 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -23,6 +23,7 @@ #include "common/common/backoff_strategy.h" #include "common/common/hash.h" #include "common/common/hex.h" +#include "common/common/utility.h" #include "common/grpc/common.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" @@ -217,13 +218,13 @@ class Utility { */ template static Factory& getAndCheckFactoryByName(const std::string& name) { if (name.empty()) { - throw EnvoyException("Provided name for static registration lookup was empty."); + ExceptionUtil::throwEnvoyException("Provided name for static registration lookup was empty."); } Factory* factory = Registry::FactoryRegistry::getFactory(name); if (factory == nullptr) { - throw EnvoyException( + ExceptionUtil::throwEnvoyException( fmt::format("Didn't find a registered implementation for name: '{}'", name)); } @@ -399,11 +400,12 @@ class Utility { const char* filter_chain_type, bool is_terminal_filter, bool last_filter_in_current_config) { if (is_terminal_filter && !last_filter_in_current_config) { - throw EnvoyException(fmt::format("Error: terminal filter named {} of type {} must be the " - "last filter in a {} filter chain.", - name, filter_type, filter_chain_type)); + ExceptionUtil::throwEnvoyException( + fmt::format("Error: terminal filter named {} of type {} must be the " + "last filter in a {} filter chain.", + name, filter_type, filter_chain_type)); } else if (!is_terminal_filter && last_filter_in_current_config) { - throw EnvoyException(fmt::format( + ExceptionUtil::throwEnvoyException(fmt::format( "Error: non-terminal filter named {} of type {} is the last filter in a {} filter chain.", name, filter_type, filter_chain_type)); } @@ -425,8 +427,9 @@ class Utility { uint64_t max_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config.dns_failure_refresh_rate(), max_interval, base_interval_ms * 10); if (max_interval_ms < base_interval_ms) { - throw EnvoyException("dns_failure_refresh_rate must have max_interval greater than " - "or equal to the base_interval"); + ExceptionUtil::throwEnvoyException( + "dns_failure_refresh_rate must have max_interval greater than " + "or equal to the base_interval"); } return std::make_unique(base_interval_ms, max_interval_ms, random); } diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 23ccee3b5ff7..cf0ded8373e9 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -136,6 +136,7 @@ envoy_cc_library( ":libevent_lib", "//include/envoy/event:timer_interface", "//source/common/common:scope_tracker", + "//source/common/common:utility_lib", "//source/common/runtime:runtime_features_lib", ], ) diff --git a/source/common/event/timer_impl.h b/source/common/event/timer_impl.h index 307fb3fe80d7..cb4e25c6fb4f 100644 --- a/source/common/event/timer_impl.h +++ b/source/common/event/timer_impl.h @@ -5,6 +5,7 @@ #include "envoy/event/timer.h" #include "common/common/scope_tracker.h" +#include "common/common/utility.h" #include "common/event/event_impl_base.h" #include "common/event/libevent.h" @@ -29,7 +30,7 @@ class TimerUtils { */ template static void durationToTimeval(const Duration& d, timeval& tv) { if (d.count() < 0) { - throw EnvoyException( + ExceptionUtil::throwEnvoyException( fmt::format("Negative duration passed to durationToTimeval(): {}", d.count())); }; constexpr int64_t clip_to = INT32_MAX; // 136.102208 years diff --git a/source/common/network/lc_trie.h b/source/common/network/lc_trie.h index dea6c6b928d8..6069eab01eb0 100644 --- a/source/common/network/lc_trie.h +++ b/source/common/network/lc_trie.h @@ -9,6 +9,7 @@ #include "envoy/network/address.h" #include "common/common/assert.h" +#include "common/common/utility.h" #include "common/network/address_impl.h" #include "common/network/cidr_range.h" #include "common/network/utility.h" @@ -68,10 +69,11 @@ template class LcTrie { } const size_t max_prefixes = MaxLcTrieNodes * fill_factor / 2; if (num_prefixes > max_prefixes) { - throw EnvoyException(fmt::format("The input vector has '{0}' CIDR range entries. LC-Trie " - "can only support '{1}' CIDR ranges with the specified " - "fill factor.", - num_prefixes, max_prefixes)); + ExceptionUtil::throwEnvoyException( + fmt::format("The input vector has '{0}' CIDR range entries. LC-Trie " + "can only support '{1}' CIDR ranges with the specified " + "fill factor.", + num_prefixes, max_prefixes)); } // Step 1: separate the provided prefixes by protocol (IPv4 vs IPv6), @@ -575,7 +577,7 @@ template class LcTrie { // number of supported trie_ entries, throw an Envoy Exception. if (position >= MaxLcTrieNodes) { // Adding 1 to the position to count how many nodes are trying to be set. - throw EnvoyException( + ExceptionUtil::throwEnvoyException( fmt::format("The number of internal nodes required for the LC-Trie " "exceeded the maximum number of " "supported nodes. Minimum number of internal nodes required: " From 1c8653f90f0c8f1f756b18e50c0d0dcbd8a86896 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 11 Aug 2020 16:38:35 -0700 Subject: [PATCH 35/67] repokitteh: fix azp retest (#12589) Signed-off-by: Lizan Zhou --- ci/repokitteh/modules/azure_pipelines.star | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/repokitteh/modules/azure_pipelines.star b/ci/repokitteh/modules/azure_pipelines.star index dc619e06d226..7d80c149b5cd 100644 --- a/ci/repokitteh/modules/azure_pipelines.star +++ b/ci/repokitteh/modules/azure_pipelines.star @@ -17,7 +17,8 @@ def _get_azp_checks(): check_ids = [] checks = [] for check in github_checks: - if check["app"]["slug"] == "azure-pipelines" and check["external_id"] not in check_ids: + # Filter out job level GitHub check, which is not individually retriable. + if check["app"]["slug"] == "azure-pipelines" and "jobId" not in check["details_url"] and check["external_id"] not in check_ids: check_ids.append(check["external_id"]) checks.append(check) From 2a4f7dae855ef81376a8aee6201ea19ea36ef8e2 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 11 Aug 2020 17:58:00 -0700 Subject: [PATCH 36/67] docs: clarify xDS resource instance version semantics (#12580) Clarify that resource instance version can be reused across stream restarts. Also a few other small fixes and improvements. Risk Level: Low Testing: N/A Docs Changes: Included in PR Release Notes: N/A Signed-off-by: Mark D. Roth --- api/xds_protocol.rst | 86 ++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/api/xds_protocol.rst b/api/xds_protocol.rst index 1b254ad7f1f3..725c5cb01564 100644 --- a/api/xds_protocol.rst +++ b/api/xds_protocol.rst @@ -226,20 +226,67 @@ Transport API version In addition the resource type version described above, the xDS wire protocol has a transport version associated with it. This provides type versioning for messages such as :ref:`DiscoveryRequest ` and :ref:`DiscoveryResponse -`. +`. It is also encoded in the gRPC method name, so a server +can determine which version a client is speaking based on which method it calls. -ACK/NACK and resource instance versioning -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Basic Protocol Overview +^^^^^^^^^^^^^^^^^^^^^^^ + +Each xDS stream begins with a :ref:`DiscoveryRequest ` from the +client, which specifies the list of resources to subscribe to, the type URL corresponding to the +subscribed resources, the node identifier, and an optional resource type instance version +indicating the most recent version of the resource type that the client has already seen (see +:ref:`ACK/NACK and resource type instance version ` for details). + +The server will then send a :ref:`DiscoveryResponse ` containing +any resources that the client has subscribed to that have changed since the last resource type +instance version that the client indicated it has seen. The server may send additional responses +at any time when the subscribed resources change. + +Whenever the client receives a new response, it will send another request indicating whether or +not the resources in the response were valid (see +:ref:`ACK/NACK and resource type instance version ` for details). + +Only the first request on a stream is guaranteed to carry the node identifier. +The subsequent discovery requests on the same stream may carry an empty node +identifier. This holds true regardless of the acceptance of the discovery +responses on the same stream. The node identifier should always be identical if +present more than once on the stream. It is sufficient to only check the first +message for the node identifier as a result. + +.. _xds_ack_nack: + +ACK/NACK and resource type instance version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Each xDS stream begins with a -:ref:`DiscoveryRequest ` from the client, specifying -the list of resources to subscribe to, the type URL corresponding to the -subscribed resources, the node identifier and an empty :ref:`version_info `. +Every xDS resource type has a version string that indicates the version for that resource type. +Whenever one resource of that type changes, the version is changed. -In addition to resource and transport type versioning schemes above, which operate at the type -level, Envoy has a resource instance version. Unlike the resource/transport types, this is not a -property of the API but is instead a reflection of the specific revision of a named resource -delivered over xDS. +In a responses sent by the xDS server, the +:ref:`version_info` field indicates the current +version for that resource type. The client then sends another request to the server with the +:ref:`version_info` field indicating the most +recent valid version seen by the client. This provides a way for the server to determine when +it sends a version that the client considers invalid. + +(In the :ref:`incremental protocol variants `, the resource type instance +version is sent by the server in the +:ref:`system_version_info` field. +However, this information is not actually used by the client to communicate which resources are +valid, because the incremental API variants have a separate mechanism for that.) + +The resource type instance version is separate for each resource type. When using the aggregated +protocol variants, each resource type has its own version even though all resource types are being +sent on the same stream. + +The resource type is also separate for each xDS server (where an xDS server is identified by a +unique :ref:`ConfigSource `). When obtaining resources of a +given type from multiple xDS servers, each xDS server will have a different notion of version. + +Note that the version for a resource type is not a property of an individual xDS stream but rather +a property of the resources themselves. If the stream becomes broken and the client creates a new +stream, the client's initial request on the new stream should indicate the most recent version +seen by the client on the previous stream. An example EDS request might be: @@ -275,7 +322,7 @@ ACK ^^^ If the update was successfully applied, the -:ref:`version_info ` will be **X**, as indicated +:ref:`version_info ` will be **X**, as indicated in the sequence diagram: .. figure:: diagrams/simple-ack.svg @@ -320,21 +367,6 @@ ACK and NACK semantics summary :ref:`version_info `. - Only the NACK should populate the :ref:`error_detail `. -Versioning and Node Identifier -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Each stream has its own notion of versioning, there is no shared -versioning across resource types. When ADS is not used, even each -resource of a given resource type may have a distinct version, since the -Envoy API allows distinct EDS/RDS resources to point at different :ref:`ConfigSources `. - -Only the first request on a stream is guaranteed to carry the node identifier. -The subsequent discovery requests on the same stream may carry an empty node -identifier. This holds true regardless of the acceptance of the discovery -responses on the same stream. The node identifier should always be identical if -present more than once on the stream. It is sufficient to only check the first -message for the node identifier as a result. - .. _xds_protocol_resource_update: When to send an update From d95d956fd0a304d06721b7ce24409bc7be30084f Mon Sep 17 00:00:00 2001 From: DongRyeol Cha Date: Wed, 12 Aug 2020 22:00:43 +0900 Subject: [PATCH 37/67] Add the bazel output file to gitignore (#12603) Commit Message: The bazel.output.txt file is created by bazel build but it is not a part of source code. It is better to be ignored. Signed-off-by: DongRyeol Cha dr83.cha@samsung.com Additional Description: None Risk Level: Low Testing: git status Docs Changes: None Release Notes: None Signed-off-by: DongRyeol Cha --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 134967bc2bb7..8884ac50e344 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ user.bazelrc CMakeLists.txt cmake-build-debug /linux +bazel.output.txt From 960ad211c3d1fe0d8838d78e0e88a5c89781f137 Mon Sep 17 00:00:00 2001 From: Sam Flattery <44659644+samflattery@users.noreply.github.com> Date: Wed, 12 Aug 2020 15:26:13 +0100 Subject: [PATCH 38/67] fuzz: change expect_* to assert in xds_fuzz_test (#12567) * the EXPECT_EQ / EXPECT_TRUE calls cause errors to be logged but the fuzzer does not crash, meaning discrepancies do not show up on oss-fuzz * removed the updateRoute() function from the verifier since it is redundant Signed-off-by: Sam Flattery --- test/server/config_validation/BUILD | 1 + test/server/config_validation/xds_fuzz.cc | 22 +++++++++---------- test/server/config_validation/xds_fuzz.h | 1 + test/server/config_validation/xds_verifier.cc | 20 ----------------- test/server/config_validation/xds_verifier.h | 1 - .../config_validation/xds_verifier_test.cc | 4 ++-- 6 files changed, 15 insertions(+), 34 deletions(-) diff --git a/test/server/config_validation/BUILD b/test/server/config_validation/BUILD index 3a710f8ff42e..e21acf9b5d00 100644 --- a/test/server/config_validation/BUILD +++ b/test/server/config_validation/BUILD @@ -145,6 +145,7 @@ envoy_cc_test_library( deps = [ ":xds_fuzz_proto_cc_proto", ":xds_verifier_lib", + "//test/fuzz:utility_lib", "//test/integration:http_integration_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/api/v2:pkg_cc_proto", diff --git a/test/server/config_validation/xds_fuzz.cc b/test/server/config_validation/xds_fuzz.cc index 9c1647c155cf..56af058905f8 100644 --- a/test/server/config_validation/xds_fuzz.cc +++ b/test/server/config_validation/xds_fuzz.cc @@ -328,13 +328,13 @@ void XdsFuzzTest::verifyListeners() { // the state should match switch (rep.state) { case XdsVerifier::DRAINING: - EXPECT_TRUE(listener_dump->has_draining_state()); + FUZZ_ASSERT(listener_dump->has_draining_state()); break; case XdsVerifier::WARMING: - EXPECT_TRUE(listener_dump->has_warming_state()); + FUZZ_ASSERT(listener_dump->has_warming_state()); break; case XdsVerifier::ACTIVE: - EXPECT_TRUE(listener_dump->has_active_state()); + FUZZ_ASSERT(listener_dump->has_active_state()); break; default: NOT_REACHED_GCOVR_EXCL_LINE; @@ -347,9 +347,9 @@ void XdsFuzzTest::verifyRoutes() { // go through routes in verifier and make sure each is in the config dump auto routes = verifier_.routes(); - EXPECT_EQ(routes.size(), dump.size()); + FUZZ_ASSERT(routes.size() == dump.size()); for (const auto& route : routes) { - EXPECT_TRUE(std::any_of(dump.begin(), dump.end(), [&](const auto& dump_route) { + FUZZ_ASSERT(std::any_of(dump.begin(), dump.end(), [&](const auto& dump_route) { return route.first == dump_route.name(); })); } @@ -361,12 +361,12 @@ void XdsFuzzTest::verifyState() { verifyRoutes(); ENVOY_LOG_MISC(debug, "Verified routes"); - EXPECT_EQ(test_server_->gauge("listener_manager.total_listeners_draining")->value(), - verifier_.numDraining()); - EXPECT_EQ(test_server_->gauge("listener_manager.total_listeners_warming")->value(), - verifier_.numWarming()); - EXPECT_EQ(test_server_->gauge("listener_manager.total_listeners_active")->value(), - verifier_.numActive()); + FUZZ_ASSERT(test_server_->gauge("listener_manager.total_listeners_draining")->value() == + verifier_.numDraining()); + FUZZ_ASSERT(test_server_->gauge("listener_manager.total_listeners_warming")->value() == + verifier_.numWarming()); + FUZZ_ASSERT(test_server_->gauge("listener_manager.total_listeners_active")->value() == + verifier_.numActive()); ENVOY_LOG_MISC(debug, "Verified stats"); ENVOY_LOG_MISC(debug, "warming {} ({}), active {} ({}), draining {} ({})", verifier_.numWarming(), test_server_->gauge("listener_manager.total_listeners_warming")->value(), diff --git a/test/server/config_validation/xds_fuzz.h b/test/server/config_validation/xds_fuzz.h index 826175bf0241..a36d332ebc65 100644 --- a/test/server/config_validation/xds_fuzz.h +++ b/test/server/config_validation/xds_fuzz.h @@ -11,6 +11,7 @@ #include "test/common/grpc/grpc_client_integration.h" #include "test/config/utility.h" +#include "test/fuzz/utility.h" #include "test/integration/http_integration.h" #include "test/server/config_validation/xds_fuzz.pb.h" #include "test/server/config_validation/xds_verifier.h" diff --git a/test/server/config_validation/xds_verifier.cc b/test/server/config_validation/xds_verifier.cc index 2501911a75d6..5623ceed35a3 100644 --- a/test/server/config_validation/xds_verifier.cc +++ b/test/server/config_validation/xds_verifier.cc @@ -265,26 +265,6 @@ void XdsVerifier::markForRemoval(ListenerRepresentation& rep) { } } -/** - * called when a route that was previously added is re-added - * the original route might have been ignored if no resources refer to it, so we can add it here - */ -void XdsVerifier::routeUpdated(const envoy::config::route::v3::RouteConfiguration& route) { - if (!all_routes_.contains(route.name()) && - std::any_of(listeners_.begin(), listeners_.end(), - [&](auto& rep) { return getRoute(rep.listener) == route.name(); })) { - all_routes_.insert({route.name(), route}); - active_routes_.insert({route.name(), route}); - } - - ENVOY_LOG_MISC(debug, "Updating {}", route.name()); - if (sotw_or_delta_ == DELTA) { - updateDeltaListeners(route); - } else { - updateSotwListeners(); - } -} - /** * add a new route and update any listeners that refer to this route */ diff --git a/test/server/config_validation/xds_verifier.h b/test/server/config_validation/xds_verifier.h index ffd7ff38231b..d19ad8f6c1ab 100644 --- a/test/server/config_validation/xds_verifier.h +++ b/test/server/config_validation/xds_verifier.h @@ -24,7 +24,6 @@ class XdsVerifier { void drainedListener(const std::string& name); void routeAdded(const envoy::config::route::v3::RouteConfiguration& route); - void routeUpdated(const envoy::config::route::v3::RouteConfiguration& route); enum ListenerState { WARMING, ACTIVE, DRAINING, REMOVED }; struct ListenerRepresentation { diff --git a/test/server/config_validation/xds_verifier_test.cc b/test/server/config_validation/xds_verifier_test.cc index 72ca229d05ba..8dc1a28d2210 100644 --- a/test/server/config_validation/xds_verifier_test.cc +++ b/test/server/config_validation/xds_verifier_test.cc @@ -187,7 +187,7 @@ TEST(XdsVerifier, ResendRouteSOTW) { EXPECT_TRUE(verifier.hasListener("listener_0", verifier.WARMING)); // send the same route again, make sure listener becomes active - verifier.routeUpdated(buildRoute("route_config_0")); + verifier.routeAdded(buildRoute("route_config_0")); EXPECT_TRUE(verifier.hasListener("listener_0", verifier.ACTIVE)); } @@ -202,7 +202,7 @@ TEST(XdsVerifier, ResendRouteDelta) { EXPECT_TRUE(verifier.hasListener("listener_0", verifier.WARMING)); // send the same route again, make sure listener becomes active - verifier.routeUpdated(buildRoute("route_config_0")); + verifier.routeAdded(buildRoute("route_config_0")); EXPECT_TRUE(verifier.hasListener("listener_0", verifier.ACTIVE)); } From 28df333572e71fc6850e82437a59e9e378d7b00b Mon Sep 17 00:00:00 2001 From: Alex Konradi Date: Wed, 12 Aug 2020 11:33:28 -0400 Subject: [PATCH 39/67] server: add support for range trigger overload actions (#12352) Change the Overload Manager API to extend the overload action state with non-binary values. This will allow future overload actions to take effect in response to increasing load, instead of to the existing inactive/active binary values. This PR also adds a range field to the overload trigger config, though no actions currently trigger on the 'scaling' state. The existing behavior is preserved by replacing all places where OverloadActionState::Active was being used with OverloadActionState::saturated(). This is a minimal re-hashing of #11697 for ##11427. Risk Level: medium Testing: ran unit and integration tests Docs Changes: none Release Notes: add "scaling" trigger for OverloadManager actions Signed-off-by: Alex Konradi --- api/envoy/config/overload/v3/overload.proto | 20 ++- .../overload_manager/overload_manager.rst | 24 +++- .../operations/overload_manager.rst | 41 ++++++ docs/root/version_history/current.rst | 1 + .../envoy/config/overload/v3/overload.proto | 20 ++- include/envoy/server/overload_manager.h | 30 ++-- source/common/http/conn_manager_impl.cc | 5 +- source/common/memory/heap_shrinker.cc | 7 +- source/server/admin/admin.h | 2 +- source/server/overload_manager_impl.cc | 108 ++++++++++---- source/server/overload_manager_impl.h | 11 +- source/server/worker_impl.cc | 7 +- test/common/http/conn_manager_impl_test.cc | 4 +- test/common/memory/heap_shrinker_test.cc | 4 +- test/mocks/server/overload_manager.cc | 2 +- test/server/overload_manager_impl_test.cc | 136 ++++++++++++++++-- 16 files changed, 342 insertions(+), 80 deletions(-) diff --git a/api/envoy/config/overload/v3/overload.proto b/api/envoy/config/overload/v3/overload.proto index d564e0d0ae3d..061783a04b77 100644 --- a/api/envoy/config/overload/v3/overload.proto +++ b/api/envoy/config/overload/v3/overload.proto @@ -50,10 +50,20 @@ message ThresholdTrigger { "envoy.config.overload.v2alpha.ThresholdTrigger"; // If the resource pressure is greater than or equal to this value, the trigger - // will fire. + // will enter saturation. double value = 1 [(validate.rules).double = {lte: 1.0 gte: 0.0}]; } +message ScaledTrigger { + // If the resource pressure is greater than this value, the trigger will be in the + // :ref:`scaling ` state with value + // `(pressure - scaling_threshold) / (saturation_threshold - scaling_threshold)`. + double scaling_threshold = 1 [(validate.rules).double = {lte: 1.0 gte: 0.0}]; + + // If the resource pressure is greater than this value, the trigger will enter saturation. + double saturation_threshold = 2 [(validate.rules).double = {lte: 1.0 gte: 0.0}]; +} + message Trigger { option (udpa.annotations.versioning).previous_message_type = "envoy.config.overload.v2alpha.Trigger"; @@ -65,6 +75,8 @@ message Trigger { option (validate.required) = true; ThresholdTrigger threshold = 2; + + ScaledTrigger scaled = 3; } } @@ -77,9 +89,9 @@ message OverloadAction { // DNS to ensure uniqueness. string name = 1 [(validate.rules).string = {min_bytes: 1}]; - // A set of triggers for this action. If any of these triggers fire the overload action - // is activated. Listeners are notified when the overload action transitions from - // inactivated to activated, or vice versa. + // A set of triggers for this action. The state of the action is the maximum + // state of all triggers, which can be scaling between 0 and 1 or saturated. Listeners + // are notified when the overload action changes state. repeated Trigger triggers = 2 [(validate.rules).repeated = {min_items: 1}]; } diff --git a/docs/root/configuration/operations/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst index 2dd2e7fe5cc7..8497f5292f32 100644 --- a/docs/root/configuration/operations/overload_manager/overload_manager.rst +++ b/docs/root/configuration/operations/overload_manager/overload_manager.rst @@ -40,6 +40,27 @@ The overload manager uses Envoy's :ref:`extension ` framework for def resource monitors. Envoy's builtin resource monitors are listed :ref:`here `. +Triggers +-------- + +Triggers connect resource monitors to actions. There are two types of triggers supported: + +.. list-table:: + :header-rows: 1 + :widths: 1, 2 + + * - Type + - Description + * - :ref:`threshold ` + - Sets the action state to 1 (= *saturated*) when the resource pressure is above a threshold, and to 0 otherwise. + * - :ref:`scaled ` + - Sets the action state to 0 when the resource pressure is below the + :ref:`scaling_threshold `, + `(pressure - scaling_threshold)/(saturation_threshold - scaling_threshold)` when + `scaling_threshold < pressure < saturation_threshold`, and to 1 (*saturated*) when the + pressure is above the + :ref:`saturation_threshold `." + Overload actions ---------------- @@ -99,4 +120,5 @@ with the following statistics: :header: Name, Type, Description :widths: 1, 1, 2 - active, Gauge, "Active state of the action (0=inactive, 1=active)" + active, Gauge, "Active state of the action (0=scaling, 1=saturated)" + scale_percent, Gauge, "Scaled value of the action as a percent (0-99=scaling, 100=saturated)" diff --git a/docs/root/intro/arch_overview/operations/overload_manager.rst b/docs/root/intro/arch_overview/operations/overload_manager.rst index af00c5948bcb..256052bde184 100644 --- a/docs/root/intro/arch_overview/operations/overload_manager.rst +++ b/docs/root/intro/arch_overview/operations/overload_manager.rst @@ -12,3 +12,44 @@ upstream services. The overload manager is :ref:`configured ` by specifying a set of resources to monitor and a set of overload actions that will be taken when some of those resources exceed certain pressure thresholds. + +Architecture +------------ + +The overload manager works by periodically polling the *pressure* of a set of **resources**, +feeding those through **triggers**, and taking **actions** based on the triggers. The set of +resource monitors, triggers, and actions are specified at startup. + +Resources +~~~~~~~~~ + +A resource is a thing that can be monitored by the overload manager, and whose *pressure* is +represented by a real value in the range [0, 1]. The pressure of a resource is evaluated by a +*resource monitor*. See the :ref:`configuration page ` for setting up +resource monitors. + +Triggers +~~~~~~~~ + +Triggers are evaluated on each resource pressure update, and convert a resource pressure value +into an action state. An action state has a value in the range [0, 1], and is categorized into one of two groups: + +.. _arch_overview_overload_manager-triggers-state: + +.. csv-table:: + :header: action state, value, description + :widths: 1, 1, 2 + + scaling, "[0, 1)", the resource pressure is below the configured saturation point; action may be taken + saturated, 1, the resource pressure is at or above the configured saturation point; drastic action should be taken + +When a resource pressure value is updated, the relevant triggers are reevaluated. For each action +with at least one trigger, the resulting action state is the maximum value over the configured +triggers. What effect the action state has depends on the action's configuration and implementation. + +Actions +~~~~~~~ + +When a trigger changes state, the value is sent to registered actions, which can then affect how +connections and requests are processed. Each action interprets the input states differently, and +some may ignore the *scaling* state altogether, taking effect only when *saturated*. \ No newline at end of file diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index ae34f7c3d119..81d351831c25 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -58,6 +58,7 @@ New Features * http: introduced new HTTP/1 and HTTP/2 codec implementations that will remove the use of exceptions for control flow due to high risk factors and instead use error statuses. The old behavior is used by default, but the new codecs can be enabled for testing by setting the runtime feature `envoy.reloadable_features.new_codec_behavior` to true. The new codecs will be in development for one month, and then enabled by default while the old codecs are deprecated. * load balancer: added a :ref:`configuration` option to specify the active request bias used by the least request load balancer. * lua: added Lua APIs to access :ref:`SSL connection info ` object. +* overload management: add :ref:`scaling ` trigger for OverloadManager actions. * postgres network filter: :ref:`metadata ` is produced based on SQL query. * ratelimit: added :ref:`enable_x_ratelimit_headers ` option to enable `X-RateLimit-*` headers as defined in `draft RFC `_. * rbac filter: added a log action to the :ref:`RBAC filter ` which sets dynamic metadata to inform access loggers whether to log. diff --git a/generated_api_shadow/envoy/config/overload/v3/overload.proto b/generated_api_shadow/envoy/config/overload/v3/overload.proto index 337150657b14..7c32906d142b 100644 --- a/generated_api_shadow/envoy/config/overload/v3/overload.proto +++ b/generated_api_shadow/envoy/config/overload/v3/overload.proto @@ -48,10 +48,20 @@ message ThresholdTrigger { "envoy.config.overload.v2alpha.ThresholdTrigger"; // If the resource pressure is greater than or equal to this value, the trigger - // will fire. + // will enter saturation. double value = 1 [(validate.rules).double = {lte: 1.0 gte: 0.0}]; } +message ScaledTrigger { + // If the resource pressure is greater than this value, the trigger will be in the + // :ref:`scaling ` state with value + // `(pressure - scaling_threshold) / (saturation_threshold - scaling_threshold)`. + double scaling_threshold = 1 [(validate.rules).double = {lte: 1.0 gte: 0.0}]; + + // If the resource pressure is greater than this value, the trigger will enter saturation. + double saturation_threshold = 2 [(validate.rules).double = {lte: 1.0 gte: 0.0}]; +} + message Trigger { option (udpa.annotations.versioning).previous_message_type = "envoy.config.overload.v2alpha.Trigger"; @@ -63,6 +73,8 @@ message Trigger { option (validate.required) = true; ThresholdTrigger threshold = 2; + + ScaledTrigger scaled = 3; } } @@ -75,9 +87,9 @@ message OverloadAction { // DNS to ensure uniqueness. string name = 1 [(validate.rules).string = {min_bytes: 1}]; - // A set of triggers for this action. If any of these triggers fire the overload action - // is activated. Listeners are notified when the overload action transitions from - // inactivated to activated, or vice versa. + // A set of triggers for this action. The state of the action is the maximum + // state of all triggers, which can be scaling between 0 and 1 or saturated. Listeners + // are notified when the overload action changes state. repeated Trigger triggers = 2 [(validate.rules).repeated = {min_items: 1}]; } diff --git a/include/envoy/server/overload_manager.h b/include/envoy/server/overload_manager.h index 24ddd16cfd6c..77c6c21c097d 100644 --- a/include/envoy/server/overload_manager.h +++ b/include/envoy/server/overload_manager.h @@ -11,15 +11,27 @@ namespace Envoy { namespace Server { -enum class OverloadActionState { - /** - * Indicates that an overload action is active because at least one of its triggers has fired. - */ - Active, - /** - * Indicates that an overload action is inactive because none of its triggers have fired. - */ - Inactive +/** + * Tracks the state of an overload action. The state is a number between 0 and 1 that represents the + * level of saturation. The values are categorized in two groups: + * - Saturated (value = 1): indicates that an overload action is active because at least one of its + * triggers has reached saturation. + * - Scaling (0 <= value < 1): indicates that an overload action is not saturated. + */ +class OverloadActionState { +public: + static constexpr OverloadActionState inactive() { return OverloadActionState(0); } + + static constexpr OverloadActionState saturated() { return OverloadActionState(1.0); } + + explicit constexpr OverloadActionState(float value) + : action_value_(std::min(1.0f, std::max(0.0f, value))) {} + + float value() const { return action_value_; } + bool isSaturated() const { return action_value_ == 1; } + +private: + float action_value_; }; /** diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index f00afb45cf7b..5d82151074f4 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -862,8 +862,7 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapPtr&& he filter_manager_.maybeEndDecode(end_stream); // Drop new requests when overloaded as soon as we have decoded the headers. - if (connection_manager_.overload_stop_accepting_requests_ref_ == - Server::OverloadActionState::Active) { + if (connection_manager_.overload_stop_accepting_requests_ref_.isSaturated()) { // In this one special case, do not create the filter chain. If there is a risk of memory // overload it is more important to avoid unnecessary allocation than to create the filters. filter_manager_.skipFilterChainCreation(); @@ -1860,7 +1859,7 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ResponseHeaderMap& heade } if (connection_manager_.drain_state_ == DrainState::NotDraining && - connection_manager_.overload_disable_keepalive_ref_ == Server::OverloadActionState::Active) { + connection_manager_.overload_disable_keepalive_ref_.isSaturated()) { ENVOY_STREAM_LOG(debug, "disabling keepalive due to envoy overload", *this); connection_manager_.drain_state_ = DrainState::Closing; connection_manager_.stats_.named_.downstream_cx_overload_disable_keepalive_.inc(); diff --git a/source/common/memory/heap_shrinker.cc b/source/common/memory/heap_shrinker.cc index da0413bbfce3..5df99c8b42d0 100644 --- a/source/common/memory/heap_shrinker.cc +++ b/source/common/memory/heap_shrinker.cc @@ -15,10 +15,9 @@ HeapShrinker::HeapShrinker(Event::Dispatcher& dispatcher, Server::OverloadManage Stats::Scope& stats) : active_(false) { const auto action_name = Server::OverloadActionNames::get().ShrinkHeap; - if (overload_manager.registerForAction(action_name, dispatcher, - [this](Server::OverloadActionState state) { - active_ = (state == Server::OverloadActionState::Active); - })) { + if (overload_manager.registerForAction( + action_name, dispatcher, + [this](Server::OverloadActionState state) { active_ = state.isSaturated(); })) { Envoy::Stats::StatNameManagedStorage stat_name( absl::StrCat("overload.", action_name, ".shrink_count"), stats.symbolTable()); shrink_counter_ = &stats.counterFromStatName(stat_name.statName()); diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index e3c66660fdc5..f2446779f952 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -256,7 +256,7 @@ class AdminImpl : public Admin, struct NullThreadLocalOverloadState : public ThreadLocalOverloadState { const OverloadActionState& getState(const std::string&) override { return inactive_; } - const OverloadActionState inactive_ = OverloadActionState::Inactive; + const OverloadActionState inactive_ = OverloadActionState::inactive(); }; NullOverloadManager(ThreadLocal::SlotAllocator& slot_allocator) diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index 1e4e085bb890..529bef09a162 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -1,5 +1,6 @@ #include "server/overload_manager_impl.h" +#include "envoy/common/exception.h" #include "envoy/config/overload/v3/overload.pb.h" #include "envoy/stats/scope.h" @@ -18,22 +19,55 @@ namespace Server { namespace { -class ThresholdTriggerImpl : public OverloadAction::Trigger { +class ThresholdTriggerImpl final : public OverloadAction::Trigger { public: ThresholdTriggerImpl(const envoy::config::overload::v3::ThresholdTrigger& config) - : threshold_(config.value()) {} + : threshold_(config.value()), state_(OverloadActionState::inactive()) {} bool updateValue(double value) override { - const bool fired = isFired(); - value_ = value; - return fired != isFired(); + const OverloadActionState state = actionState(); + state_ = + value >= threshold_ ? OverloadActionState::saturated() : OverloadActionState::inactive(); + return state.value() != actionState().value(); } - bool isFired() const override { return value_.has_value() && value_ >= threshold_; } + OverloadActionState actionState() const override { return state_; } private: const double threshold_; - absl::optional value_; + OverloadActionState state_; +}; + +class ScaledTriggerImpl final : public OverloadAction::Trigger { +public: + ScaledTriggerImpl(const envoy::config::overload::v3::ScaledTrigger& config) + : scaling_threshold_(config.scaling_threshold()), + saturated_threshold_(config.saturation_threshold()), + state_(OverloadActionState::inactive()) { + if (scaling_threshold_ >= saturated_threshold_) { + throw EnvoyException("scaling_threshold must be less than saturation_threshold"); + } + } + + bool updateValue(double value) override { + const OverloadActionState old_state = actionState(); + if (value <= scaling_threshold_) { + state_ = OverloadActionState::inactive(); + } else if (value >= saturated_threshold_) { + state_ = OverloadActionState::saturated(); + } else { + state_ = OverloadActionState((value - scaling_threshold_) / + (saturated_threshold_ - scaling_threshold_)); + } + return state_.value() != old_state.value(); + } + + OverloadActionState actionState() const override { return state_; } + +private: + const double scaling_threshold_; + const double saturated_threshold_; + OverloadActionState state_; }; /** @@ -44,12 +78,14 @@ class ThreadLocalOverloadStateImpl : public ThreadLocalOverloadState { const OverloadActionState& getState(const std::string& action) override { auto it = actions_.find(action); if (it == actions_.end()) { - it = actions_.insert(std::make_pair(action, OverloadActionState::Inactive)).first; + it = actions_.insert(std::make_pair(action, OverloadActionState::inactive())).first; } return it->second; } - void setState(const std::string& action, OverloadActionState state) { actions_[action] = state; } + void setState(const std::string& action, OverloadActionState state) { + actions_.insert_or_assign(action, state); + } private: absl::node_hash_map actions_; @@ -72,8 +108,11 @@ Stats::Gauge& makeGauge(Stats::Scope& scope, absl::string_view a, absl::string_v OverloadAction::OverloadAction(const envoy::config::overload::v3::OverloadAction& config, Stats::Scope& stats_scope) - : active_gauge_( - makeGauge(stats_scope, config.name(), "active", Stats::Gauge::ImportMode::Accumulate)) { + : state_(OverloadActionState::inactive()), + active_gauge_( + makeGauge(stats_scope, config.name(), "active", Stats::Gauge::ImportMode::Accumulate)), + scale_percent_gauge_(makeGauge(stats_scope, config.name(), "scale_percent", + Stats::Gauge::ImportMode::Accumulate)) { for (const auto& trigger_config : config.triggers()) { TriggerPtr trigger; @@ -81,6 +120,9 @@ OverloadAction::OverloadAction(const envoy::config::overload::v3::OverloadAction case envoy::config::overload::v3::Trigger::TriggerOneofCase::kThreshold: trigger = std::make_unique(trigger_config.threshold()); break; + case envoy::config::overload::v3::Trigger::TriggerOneofCase::kScaled: + trigger = std::make_unique(trigger_config.scaled()); + break; default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -92,29 +134,37 @@ OverloadAction::OverloadAction(const envoy::config::overload::v3::OverloadAction } active_gauge_.set(0); + scale_percent_gauge_.set(0); } bool OverloadAction::updateResourcePressure(const std::string& name, double pressure) { - const bool active = isActive(); + const OverloadActionState old_state = getState(); auto it = triggers_.find(name); ASSERT(it != triggers_.end()); - if (it->second->updateValue(pressure)) { - if (it->second->isFired()) { - active_gauge_.set(1); - const auto result = fired_triggers_.insert(name); - ASSERT(result.second); - } else { - active_gauge_.set(0); - const auto result = fired_triggers_.erase(name); - ASSERT(result == 1); + if (!it->second->updateValue(pressure)) { + return false; + } + const auto trigger_new_state = it->second->actionState(); + active_gauge_.set(trigger_new_state.isSaturated() ? 1 : 0); + scale_percent_gauge_.set(trigger_new_state.value() * 100); + + { + // Compute the new state as the maximum over all trigger states. + OverloadActionState new_state = OverloadActionState::inactive(); + for (auto& trigger : triggers_) { + const auto trigger_state = trigger.second->actionState(); + if (trigger_state.value() > new_state.value()) { + new_state = trigger_state; + } } + state_ = new_state; } - return active != isActive(); + return state_.value() != old_state.value(); } -bool OverloadAction::isActive() const { return !fired_triggers_.empty(); } +OverloadActionState OverloadAction::getState() const { return state_; } OverloadManagerImpl::OverloadManagerImpl(Event::Dispatcher& dispatcher, Stats::Scope& stats_scope, ThreadLocal::SlotAllocator& slot_allocator, @@ -223,12 +273,14 @@ void OverloadManagerImpl::updateResourcePressure(const std::string& resource, do const std::string& action = entry.second; auto action_it = actions_.find(action); ASSERT(action_it != actions_.end()); + const OverloadActionState old_state = action_it->second.getState(); if (action_it->second.updateResourcePressure(resource, pressure)) { - const bool is_active = action_it->second.isActive(); - const auto state = - is_active ? OverloadActionState::Active : OverloadActionState::Inactive; - ENVOY_LOG(info, "Overload action {} became {}", action, - is_active ? "active" : "inactive"); + const auto state = action_it->second.getState(); + + if (old_state.isSaturated() != state.isSaturated()) { + ENVOY_LOG(debug, "Overload action {} became {}", action, + (state.isSaturated() ? "saturated" : "scaling")); + } tls_->runOnAllThreads([this, action, state] { tls_->getTyped().setState(action, state); }); diff --git a/source/server/overload_manager_impl.h b/source/server/overload_manager_impl.h index 4405bfeaf3aa..e19a74b55a10 100644 --- a/source/server/overload_manager_impl.h +++ b/source/server/overload_manager_impl.h @@ -30,8 +30,8 @@ class OverloadAction { // has changed state. bool updateResourcePressure(const std::string& name, double pressure); - // Returns whether the action is currently active or not. - bool isActive() const; + // Returns the current action state, which is the max state across all registered triggers. + OverloadActionState getState() const; class Trigger { public: @@ -40,15 +40,16 @@ class OverloadAction { // Updates the current value of the metric and returns whether the trigger has changed state. virtual bool updateValue(double value) PURE; - // Returns whether the trigger is currently fired or not. - virtual bool isFired() const PURE; + // Returns the action state for the trigger. + virtual OverloadActionState actionState() const PURE; }; using TriggerPtr = std::unique_ptr; private: absl::node_hash_map triggers_; - absl::node_hash_set fired_triggers_; + OverloadActionState state_; Stats::Gauge& active_gauge_; + Stats::Gauge& scale_percent_gauge_; }; class OverloadManagerImpl : Logger::Loggable, public OverloadManager { diff --git a/source/server/worker_impl.cc b/source/server/worker_impl.cc index eae51fa68837..b5bbe8d91cbb 100644 --- a/source/server/worker_impl.cc +++ b/source/server/worker_impl.cc @@ -143,13 +143,10 @@ void WorkerImpl::threadRoutine(GuardDog& guard_dog) { } void WorkerImpl::stopAcceptingConnectionsCb(OverloadActionState state) { - switch (state) { - case OverloadActionState::Active: + if (state.isSaturated()) { handler_->disableListeners(); - break; - case OverloadActionState::Inactive: + } else { handler_->enableListeners(); - break; } } diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index a13dd0133a1c..af6780154c2c 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -5723,7 +5723,7 @@ TEST(HttpConnectionManagerTracingStatsTest, verifyTracingStats) { } TEST_F(HttpConnectionManagerImplTest, NoNewStreamWhenOverloaded) { - Server::OverloadActionState stop_accepting_requests = Server::OverloadActionState::Active; + Server::OverloadActionState stop_accepting_requests = Server::OverloadActionState::saturated(); ON_CALL(overload_manager_.overload_state_, getState(Server::OverloadActionNames::get().StopAcceptingRequests)) .WillByDefault(ReturnRef(stop_accepting_requests)); @@ -5754,7 +5754,7 @@ TEST_F(HttpConnectionManagerImplTest, NoNewStreamWhenOverloaded) { } TEST_F(HttpConnectionManagerImplTest, DisableKeepAliveWhenOverloaded) { - Server::OverloadActionState disable_http_keep_alive = Server::OverloadActionState::Active; + Server::OverloadActionState disable_http_keep_alive = Server::OverloadActionState::saturated(); ON_CALL(overload_manager_.overload_state_, getState(Server::OverloadActionNames::get().DisableHttpKeepAlive)) .WillByDefault(ReturnRef(disable_http_keep_alive)); diff --git a/test/common/memory/heap_shrinker_test.cc b/test/common/memory/heap_shrinker_test.cc index 5889424f5435..e23336191dc9 100644 --- a/test/common/memory/heap_shrinker_test.cc +++ b/test/common/memory/heap_shrinker_test.cc @@ -61,7 +61,7 @@ TEST_F(HeapShrinkerTest, ShrinkWhenTriggered) { Envoy::Stats::Counter& shrink_count = stats_.counter("overload.envoy.overload_actions.shrink_heap.shrink_count"); - action_cb(Server::OverloadActionState::Active); + action_cb(Server::OverloadActionState::saturated()); step(); EXPECT_EQ(1, shrink_count.value()); @@ -77,7 +77,7 @@ TEST_F(HeapShrinkerTest, ShrinkWhenTriggered) { step(); EXPECT_EQ(2, shrink_count.value()); - action_cb(Server::OverloadActionState::Inactive); + action_cb(Server::OverloadActionState::inactive()); step(); step(); EXPECT_EQ(2, shrink_count.value()); diff --git a/test/mocks/server/overload_manager.cc b/test/mocks/server/overload_manager.cc index d0fd9b545ec6..1402be606930 100644 --- a/test/mocks/server/overload_manager.cc +++ b/test/mocks/server/overload_manager.cc @@ -11,7 +11,7 @@ namespace Server { using ::testing::ReturnRef; MockThreadLocalOverloadState::MockThreadLocalOverloadState() - : disabled_state_(OverloadActionState::Inactive) { + : disabled_state_(OverloadActionState::inactive()) { ON_CALL(*this, getState).WillByDefault(ReturnRef(disabled_state_)); } diff --git a/test/server/overload_manager_impl_test.cc b/test/server/overload_manager_impl_test.cc index 73715e249c07..4ccb043e1d8d 100644 --- a/test/server/overload_manager_impl_test.cc +++ b/test/server/overload_manager_impl_test.cc @@ -1,4 +1,5 @@ #include "envoy/config/overload/v3/overload.pb.h" +#include "envoy/server/overload_manager.h" #include "envoy/server/resource_monitor.h" #include "envoy/server/resource_monitor_config.h" @@ -19,8 +20,10 @@ #include "gtest/gtest.h" using testing::_; +using testing::AllOf; using testing::Invoke; using testing::NiceMock; +using testing::Property; namespace Envoy { namespace Server { @@ -83,8 +86,10 @@ class OverloadManagerImplTest : public testing::Test { protected: OverloadManagerImplTest() : factory1_("envoy.resource_monitors.fake_resource1"), - factory2_("envoy.resource_monitors.fake_resource2"), register_factory1_(factory1_), - register_factory2_(factory2_), api_(Api::createApiForTest(stats_)) {} + factory2_("envoy.resource_monitors.fake_resource2"), + factory3_("envoy.resource_monitors.fake_resource3"), register_factory1_(factory1_), + register_factory2_(factory2_), register_factory3_(factory3_), + api_(Api::createApiForTest(stats_)) {} void setDispatcherExpectation() { timer_ = new NiceMock(); @@ -112,6 +117,9 @@ class OverloadManagerImplTest : public testing::Test { resource_monitors { name: "envoy.resource_monitors.fake_resource2" } + resource_monitors { + name: "envoy.resource_monitors.fake_resource3" + } actions { name: "envoy.overload_actions.dummy_action" triggers { @@ -126,6 +134,13 @@ class OverloadManagerImplTest : public testing::Test { value: 0.8 } } + triggers { + name: "envoy.resource_monitors.fake_resource3" + scaled { + scaling_threshold: 0.5 + saturation_threshold: 0.8 + } + } } )EOF"; } @@ -157,8 +172,10 @@ class OverloadManagerImplTest : public testing::Test { FakeResourceMonitorFactory factory1_; FakeResourceMonitorFactory factory2_; + FakeResourceMonitorFactory factory3_; Registry::InjectFactory register_factory1_; Registry::InjectFactory register_factory2_; + Registry::InjectFactory register_factory3_; NiceMock dispatcher_; NiceMock* timer_; // not owned Stats::TestUtil::TestStore stats_; @@ -176,7 +193,7 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { int cb_count = 0; manager->registerForAction("envoy.overload_actions.dummy_action", dispatcher_, [&](OverloadActionState state) { - is_active = state == OverloadActionState::Active; + is_active = state.isSaturated(); cb_count++; }); manager->registerForAction("envoy.overload_actions.unknown_action", dispatcher_, @@ -185,6 +202,9 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { Stats::Gauge& active_gauge = stats_.gauge("overload.envoy.overload_actions.dummy_action.active", Stats::Gauge::ImportMode::Accumulate); + Stats::Gauge& scale_percent_gauge = + stats_.gauge("overload.envoy.overload_actions.dummy_action.scale_percent", + Stats::Gauge::ImportMode::Accumulate); Stats::Gauge& pressure_gauge1 = stats_.gauge("overload.envoy.resource_monitors.fake_resource1.pressure", Stats::Gauge::ImportMode::NeverImport); @@ -198,25 +218,28 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { factory1_.monitor_->setPressure(0.5); timer_cb_(); EXPECT_FALSE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Inactive); + EXPECT_THAT(action_state, AllOf(Property(&OverloadActionState::isSaturated, false), + Property(&OverloadActionState::value, 0))); EXPECT_EQ(0, cb_count); EXPECT_EQ(0, active_gauge.value()); + EXPECT_EQ(0, scale_percent_gauge.value()); EXPECT_EQ(50, pressure_gauge1.value()); // Update exceeds fake_resource1 trigger threshold, callback is expected factory1_.monitor_->setPressure(0.95); timer_cb_(); EXPECT_TRUE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Active); + EXPECT_TRUE(action_state.isSaturated()); EXPECT_EQ(1, cb_count); EXPECT_EQ(1, active_gauge.value()); + EXPECT_EQ(100, scale_percent_gauge.value()); EXPECT_EQ(95, pressure_gauge1.value()); // Callback should not be invoked if action state does not change factory1_.monitor_->setPressure(0.94); timer_cb_(); EXPECT_TRUE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Active); + EXPECT_TRUE(action_state.isSaturated()); EXPECT_EQ(1, cb_count); EXPECT_EQ(94, pressure_gauge1.value()); @@ -224,7 +247,7 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { factory2_.monitor_->setPressure(0.9); timer_cb_(); EXPECT_TRUE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Active); + EXPECT_TRUE(action_state.isSaturated()); EXPECT_EQ(1, cb_count); EXPECT_EQ(90, pressure_gauge2.value()); @@ -232,7 +255,7 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { factory1_.monitor_->setPressure(0.5); timer_cb_(); EXPECT_TRUE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Active); + EXPECT_TRUE(action_state.isSaturated()); EXPECT_EQ(1, cb_count); EXPECT_EQ(50, pressure_gauge1.value()); EXPECT_EQ(90, pressure_gauge2.value()); @@ -241,7 +264,8 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { factory2_.monitor_->setPressure(0.3); timer_cb_(); EXPECT_FALSE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Inactive); + EXPECT_THAT(action_state, AllOf(Property(&OverloadActionState::isSaturated, false), + Property(&OverloadActionState::value, 0))); EXPECT_EQ(2, cb_count); EXPECT_EQ(30, pressure_gauge2.value()); @@ -250,7 +274,7 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { factory2_.monitor_->setPressure(0.96); timer_cb_(); EXPECT_TRUE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Active); + EXPECT_TRUE(action_state.isSaturated()); EXPECT_EQ(3, cb_count); EXPECT_EQ(97, pressure_gauge1.value()); EXPECT_EQ(96, pressure_gauge2.value()); @@ -260,7 +284,8 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { factory2_.monitor_->setPressure(0.42); timer_cb_(); EXPECT_FALSE(is_active); - EXPECT_EQ(action_state, OverloadActionState::Inactive); + EXPECT_THAT(action_state, AllOf(Property(&OverloadActionState::isSaturated, false), + Property(&OverloadActionState::value, 0))); EXPECT_EQ(4, cb_count); EXPECT_EQ(41, pressure_gauge1.value()); EXPECT_EQ(42, pressure_gauge2.value()); @@ -268,6 +293,51 @@ TEST_F(OverloadManagerImplTest, CallbackOnlyFiresWhenStateChanges) { manager->stop(); } +TEST_F(OverloadManagerImplTest, ScaledTrigger) { + setDispatcherExpectation(); + + auto manager(createOverloadManager(getConfig())); + manager->start(); + const auto& action_state = + manager->getThreadLocalOverloadState().getState("envoy.overload_actions.dummy_action"); + Stats::Gauge& active_gauge = stats_.gauge("overload.envoy.overload_actions.dummy_action.active", + Stats::Gauge::ImportMode::Accumulate); + Stats::Gauge& scale_percent_gauge = + stats_.gauge("overload.envoy.overload_actions.dummy_action.scale_percent", + Stats::Gauge::ImportMode::Accumulate); + + factory3_.monitor_->setPressure(0.5); + timer_cb_(); + + EXPECT_THAT(action_state, AllOf(Property(&OverloadActionState::isSaturated, false), + Property(&OverloadActionState::value, 0))); + EXPECT_EQ(0, active_gauge.value()); + EXPECT_EQ(0, scale_percent_gauge.value()); + + // The trigger for fake_resource3 is a scaled trigger with a min of 0.5 and a max of 0.8. Set the + // current pressure value to halfway in that range. + factory3_.monitor_->setPressure(0.65); + timer_cb_(); + + EXPECT_EQ(action_state.value(), 0.5 /* = 0.65 / (0.8 - 0.5) */); + EXPECT_EQ(0, active_gauge.value()); + EXPECT_EQ(50, scale_percent_gauge.value()); + + factory3_.monitor_->setPressure(0.8); + timer_cb_(); + + EXPECT_TRUE(action_state.isSaturated()); + EXPECT_EQ(1, active_gauge.value()); + EXPECT_EQ(100, scale_percent_gauge.value()); + + factory3_.monitor_->setPressure(0.9); + timer_cb_(); + + EXPECT_TRUE(action_state.isSaturated()); + EXPECT_EQ(1, active_gauge.value()); + EXPECT_EQ(100, scale_percent_gauge.value()); +} + TEST_F(OverloadManagerImplTest, FailedUpdates) { setDispatcherExpectation(); auto manager(createOverloadManager(getConfig())); @@ -339,6 +409,50 @@ TEST_F(OverloadManagerImplTest, DuplicateOverloadAction) { "Duplicate overload action .*"); } +// A scaled trigger action's thresholds must conform to scaling < saturation. +TEST_F(OverloadManagerImplTest, ScaledTriggerSaturationLessThanScalingThreshold) { + const std::string config = R"EOF( + resource_monitors { + name: "envoy.resource_monitors.fake_resource1" + } + actions { + name: "envoy.overload_actions.dummy_action" + triggers { + name: "envoy.resource_monitors.fake_resource1" + scaled { + scaling_threshold: 0.9 + saturation_threshold: 0.8 + } + } + } + )EOF"; + + EXPECT_THROW_WITH_REGEX(createOverloadManager(config), EnvoyException, + "scaling_threshold must be less than saturation_threshold.*"); +} + +// A scaled trigger action can't have threshold values that are equal. +TEST_F(OverloadManagerImplTest, ScaledTriggerThresholdsEqual) { + const std::string config = R"EOF( + resource_monitors { + name: "envoy.resource_monitors.fake_resource1" + } + actions { + name: "envoy.overload_actions.dummy_action" + triggers { + name: "envoy.resource_monitors.fake_resource1" + scaled { + scaling_threshold: 0.9 + saturation_threshold: 0.9 + } + } + } + )EOF"; + + EXPECT_THROW_WITH_REGEX(createOverloadManager(config), EnvoyException, + "scaling_threshold must be less than saturation_threshold.*"); +} + TEST_F(OverloadManagerImplTest, UnknownTrigger) { const std::string config = R"EOF( actions { From 349841315d0e935273806f3750ffc26cccd4330d Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 12 Aug 2020 09:16:23 -0700 Subject: [PATCH 40/67] vhds: handle dangling reference during VHDS update request (#12395) Signed-off-by: Dmitri Dolguikh --- source/common/router/rds_impl.cc | 13 +++++++--- source/common/router/rds_impl.h | 3 +++ test/common/router/rds_impl_test.cc | 37 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index d84eb0a3d4f5..a27238a87e0b 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -306,10 +306,17 @@ void RdsRouteConfigProviderImpl::requestVirtualHostsUpdate( std::weak_ptr route_config_updated_cb) { auto alias = VhdsSubscription::domainNameToAlias(config_update_info_->routeConfigName(), for_domain); - factory_context_.dispatcher().post([this, alias, &thread_local_dispatcher, + // The RdsRouteConfigProviderImpl instance can go away before the dispatcher has a chance to + // execute the callback. still_alive shared_ptr will be deallocated when the current instance of + // the RdsRouteConfigProviderImpl is deallocated; we rely on a weak_ptr to still_alive flag to + // determine if the RdsRouteConfigProviderImpl instance is still valid. + factory_context_.dispatcher().post([this, maybe_still_alive = std::weak_ptr(still_alive_), + alias, &thread_local_dispatcher, route_config_updated_cb]() -> void { - subscription_->updateOnDemand(alias); - config_update_callbacks_.push_back({alias, thread_local_dispatcher, route_config_updated_cb}); + if (maybe_still_alive.lock()) { + subscription_->updateOnDemand(alias); + config_update_callbacks_.push_back({alias, thread_local_dispatcher, route_config_updated_cb}); + } }); } diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index cba4793acd5a..ca9e3c562741 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -226,6 +226,9 @@ class RdsRouteConfigProviderImpl : public RouteConfigProvider, ProtobufMessage::ValidationVisitor& validator_; ThreadLocal::SlotPtr tls_; std::list config_update_callbacks_; + // A flag used to determine if this instance of RdsRouteConfigProviderImpl hasn't been + // deallocated. Please also see a comment in requestVirtualHostsUpdate() method implementation. + std::shared_ptr still_alive_{std::make_shared(true)}; friend class RouteConfigProviderManagerImpl; }; diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 3c70a9c93f54..a33629239fb8 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -33,6 +33,7 @@ using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::ReturnRef; +using testing::SaveArg; namespace Envoy { namespace Router { @@ -278,6 +279,42 @@ TEST_F(RdsImplTest, FailureSubscription) { rds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, {}); } +// Verifies that a queued up request for a virtual host update doesn't crash if +// RdsRouteConfigProvider is deallocated +TEST_F(RdsImplTest, VirtualHostUpdateWhenProviderHasBeenDeallocated) { + const std::string rds_config = R"EOF( +rds: + route_config_name: my_route + config_source: + api_config_source: + api_type: GRPC + grpc_services: + envoy_grpc: + cluster_name: xds_cluster +)EOF"; + + Event::PostCb post_cb; + testing::NiceMock local_thread_dispatcher; + testing::MockFunction mock_callback; + { + auto rds = RouteConfigProviderUtil::create( + parseHttpConnectionManagerFromYaml(rds_config), server_factory_context_, + validation_visitor_, outer_init_manager_, "foo.", *route_config_provider_manager_); + + EXPECT_CALL(server_factory_context_.dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); + rds->requestVirtualHostsUpdate( + "testing", local_thread_dispatcher, + std::make_shared( + Http::RouteConfigUpdatedCallback(mock_callback.AsStdFunction()))); + } + + // Invoke the callback that was scheduled on the main thread + // RdsRouteConfigProvider in rds is out of scope and callback's captured parameters are no longer + // valid + EXPECT_CALL(mock_callback, Call(_)).Times(0); + EXPECT_NO_THROW(post_cb()); +} + class RdsRouteConfigSubscriptionTest : public RdsTestBase { public: RdsRouteConfigSubscriptionTest() { From bcccbfa86b004d4c81636cc7840c8373cb888d47 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 12 Aug 2020 10:32:14 -0700 Subject: [PATCH 41/67] network: refactor tcp listener to use file events (#12547) IoHandle specific, implicitly, socket interface specific, implementations of listen and accept are now used to accept new IoHandles, instead of the os specific sycalls libevent leverages internally for listeners. Signed-off-by: Florin Coras --- include/envoy/api/os_sys_calls.h | 5 + include/envoy/network/io_handle.h | 9 ++ source/common/api/posix/os_sys_calls_impl.cc | 19 +++ source/common/api/posix/os_sys_calls_impl.h | 1 + source/common/api/win32/os_sys_calls_impl.cc | 10 ++ source/common/api/win32/os_sys_calls_impl.h | 1 + source/common/event/libevent.h | 1 - source/common/network/BUILD | 1 - source/common/network/base_listener_impl.h | 3 - .../common/network/io_socket_handle_impl.cc | 9 ++ source/common/network/io_socket_handle_impl.h | 1 + source/common/network/listen_socket_impl.h | 3 +- source/common/network/listener_impl.cc | 117 +++++++++--------- source/common/network/listener_impl.h | 7 +- .../quiche/quic_io_handle_wrapper.h | 3 + test/mocks/network/io_handle.h | 1 + 16 files changed, 119 insertions(+), 72 deletions(-) diff --git a/include/envoy/api/os_sys_calls.h b/include/envoy/api/os_sys_calls.h index 071a5465d5b5..24e365451837 100644 --- a/include/envoy/api/os_sys_calls.h +++ b/include/envoy/api/os_sys_calls.h @@ -160,6 +160,11 @@ class OsSysCalls { * @see man 2 write */ virtual SysCallSizeResult write(os_fd_t socket, const void* buffer, size_t length) PURE; + + /** + * @see man 2 accept. The fds returned are configured to be non-blocking. + */ + virtual SysCallSocketResult accept(os_fd_t socket, sockaddr* addr, socklen_t* addrlen) PURE; }; using OsSysCallsPtr = std::unique_ptr; diff --git a/include/envoy/network/io_handle.h b/include/envoy/network/io_handle.h index f5d18b532332..6e98a5428038 100644 --- a/include/envoy/network/io_handle.h +++ b/include/envoy/network/io_handle.h @@ -165,6 +165,15 @@ class IoHandle { */ virtual Api::SysCallIntResult listen(int backlog) PURE; + /** + * Accept on listening handle + * @param addr remote address to be returned + * @param addrlen remote address length + * @param flags flags to be applied to accepted session + * @return accepted IoHandlePtr + */ + virtual std::unique_ptr accept(struct sockaddr* addr, socklen_t* addrlen) PURE; + /** * Connect to address. The handle should have been created with a call to socket() * on this object. diff --git a/source/common/api/posix/os_sys_calls_impl.cc b/source/common/api/posix/os_sys_calls_impl.cc index 546015123bc0..64bfa7a5b517 100644 --- a/source/common/api/posix/os_sys_calls_impl.cc +++ b/source/common/api/posix/os_sys_calls_impl.cc @@ -202,5 +202,24 @@ SysCallSizeResult OsSysCallsImpl::write(os_fd_t sockfd, const void* buffer, size return {rc, rc != -1 ? 0 : errno}; } +SysCallSocketResult OsSysCallsImpl::accept(os_fd_t sockfd, sockaddr* addr, socklen_t* addrlen) { + os_fd_t rc; + +#if defined(__linux__) + rc = ::accept4(sockfd, addr, addrlen, SOCK_NONBLOCK); + // If failed with EINVAL try without flags + if (rc >= 0 || errno != EINVAL) { + return {rc, rc != -1 ? 0 : errno}; + } +#endif + + rc = ::accept(sockfd, addr, addrlen); + if (rc >= 0) { + setsocketblocking(rc, false); + } + + return {rc, rc != -1 ? 0 : errno}; +} + } // namespace Api } // namespace Envoy diff --git a/source/common/api/posix/os_sys_calls_impl.h b/source/common/api/posix/os_sys_calls_impl.h index 036604eb40c1..0238459e87b5 100644 --- a/source/common/api/posix/os_sys_calls_impl.h +++ b/source/common/api/posix/os_sys_calls_impl.h @@ -44,6 +44,7 @@ class OsSysCallsImpl : public OsSysCalls { SysCallIntResult socketpair(int domain, int type, int protocol, os_fd_t sv[2]) override; SysCallIntResult listen(os_fd_t sockfd, int backlog) override; SysCallSizeResult write(os_fd_t socket, const void* buffer, size_t length) override; + SysCallSocketResult accept(os_fd_t socket, sockaddr* addr, socklen_t* addrlen) override; }; using OsSysCallsSingleton = ThreadSafeSingleton; diff --git a/source/common/api/win32/os_sys_calls_impl.cc b/source/common/api/win32/os_sys_calls_impl.cc index 86519612a253..820704849a38 100644 --- a/source/common/api/win32/os_sys_calls_impl.cc +++ b/source/common/api/win32/os_sys_calls_impl.cc @@ -349,5 +349,15 @@ SysCallSizeResult OsSysCallsImpl::write(os_fd_t sockfd, const void* buffer, size return {rc, rc != -1 ? 0 : ::WSAGetLastError()}; } +SysCallSocketResult OsSysCallsImpl::accept(os_fd_t sockfd, sockaddr* addr, socklen_t* addrlen) { + const os_fd_t rc = ::accept(sockfd, addr, addrlen); + if (SOCKET_INVALID(rc)) { + return {rc, ::WSAGetLastError()}; + } + + setsocketblocking(rc, false); + return {rc, 0}; +} + } // namespace Api } // namespace Envoy diff --git a/source/common/api/win32/os_sys_calls_impl.h b/source/common/api/win32/os_sys_calls_impl.h index 3a2ca378d658..efc879e393e2 100644 --- a/source/common/api/win32/os_sys_calls_impl.h +++ b/source/common/api/win32/os_sys_calls_impl.h @@ -45,6 +45,7 @@ class OsSysCallsImpl : public OsSysCalls { SysCallIntResult socketpair(int domain, int type, int protocol, os_fd_t sv[2]) override; SysCallIntResult listen(os_fd_t sockfd, int backlog) override; SysCallSizeResult write(os_fd_t socket, const void* buffer, size_t length) override; + SysCallSocketResult accept(os_fd_t socket, sockaddr* addr, socklen_t* addrlen) override; }; using OsSysCallsSingleton = ThreadSafeSingleton; diff --git a/source/common/event/libevent.h b/source/common/event/libevent.h index dd7f7cd25ebb..304b326fb4dd 100644 --- a/source/common/event/libevent.h +++ b/source/common/event/libevent.h @@ -34,7 +34,6 @@ class Global { }; using BasePtr = CSmartPtr; -using ListenerPtr = CSmartPtr; } // namespace Libevent } // namespace Event diff --git a/source/common/network/BUILD b/source/common/network/BUILD index b40195b288b7..27a0d6aa2762 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -248,7 +248,6 @@ envoy_cc_library( "//source/common/common:empty_string", "//source/common/common:linked_object", "//source/common/event:dispatcher_includes", - "//source/common/event:libevent_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/common/network/base_listener_impl.h b/source/common/network/base_listener_impl.h index 878136a4fe29..2cf97dea86c4 100644 --- a/source/common/network/base_listener_impl.h +++ b/source/common/network/base_listener_impl.h @@ -3,11 +3,8 @@ #include "envoy/network/listener.h" #include "common/event/dispatcher_impl.h" -#include "common/event/libevent.h" #include "common/network/listen_socket_impl.h" -#include "event2/event.h" - namespace Envoy { namespace Network { diff --git a/source/common/network/io_socket_handle_impl.cc b/source/common/network/io_socket_handle_impl.cc index 5edd1fe5d054..8ed15b0d0253 100644 --- a/source/common/network/io_socket_handle_impl.cc +++ b/source/common/network/io_socket_handle_impl.cc @@ -387,6 +387,15 @@ Api::SysCallIntResult IoSocketHandleImpl::listen(int backlog) { return Api::OsSysCallsSingleton::get().listen(fd_, backlog); } +IoHandlePtr IoSocketHandleImpl::accept(struct sockaddr* addr, socklen_t* addrlen) { + auto result = Api::OsSysCallsSingleton::get().accept(fd_, addr, addrlen); + if (SOCKET_INVALID(result.rc_)) { + return nullptr; + } + + return std::make_unique(result.rc_, socket_v6only_); +} + Api::SysCallIntResult IoSocketHandleImpl::connect(Address::InstanceConstSharedPtr address) { return Api::OsSysCallsSingleton::get().connect(fd_, address->sockAddr(), address->sockAddrLen()); } diff --git a/source/common/network/io_socket_handle_impl.h b/source/common/network/io_socket_handle_impl.h index e23d0f444726..7b382556ce82 100644 --- a/source/common/network/io_socket_handle_impl.h +++ b/source/common/network/io_socket_handle_impl.h @@ -49,6 +49,7 @@ class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable class NetworkListenSocket : public ListenSocketImpl { public: NetworkListenSocket(const Address::InstanceConstSharedPtr& address, const Network::Socket::OptionsSharedPtr& options, bool bind_to_port) - : ListenSocketImpl(Network::SocketInterfaceSingleton::get().socket(T::type, address), - address) { + : ListenSocketImpl(Network::ioHandleForAddr(T::type, address), address) { RELEASE_ASSERT(SOCKET_VALID(io_handle_->fd()), ""); setPrebindSocketOptions(); diff --git a/source/common/network/listener_impl.cc b/source/common/network/listener_impl.cc index 96c1eded88dd..ebf2f2b7731d 100644 --- a/source/common/network/listener_impl.cc +++ b/source/common/network/listener_impl.cc @@ -14,8 +14,6 @@ #include "common/network/address_impl.h" #include "common/network/io_socket_handle_impl.h" -#include "event2/listener.h" - namespace Envoy { namespace Network { @@ -43,85 +41,84 @@ bool ListenerImpl::rejectCxOverGlobalLimit() { return AcceptedSocketImpl::acceptedSocketCount() >= global_cx_limit; } -void ListenerImpl::listenCallback(evconnlistener*, evutil_socket_t fd, sockaddr* remote_addr, - int remote_addr_len, void* arg) { - ListenerImpl* listener = static_cast(arg); - - // Wrap raw socket fd in IoHandle. - IoHandlePtr io_handle = SocketInterfaceSingleton::get().socket(fd); - - if (rejectCxOverGlobalLimit()) { - // The global connection limit has been reached. - io_handle->close(); - listener->cb_.onReject(); - return; +void ListenerImpl::onSocketEvent(short flags) { + ASSERT(flags & (Event::FileReadyType::Read)); + + // TODO(fcoras): Add limit on number of accepted calls per wakeup + while (1) { + if (!socket_->ioHandle().isOpen()) { + PANIC(fmt::format("listener accept failure: {}", errorDetails(errno))); + } + + sockaddr_storage remote_addr; + socklen_t remote_addr_len = sizeof(remote_addr); + + IoHandlePtr io_handle = + socket_->ioHandle().accept(reinterpret_cast(&remote_addr), &remote_addr_len); + if (io_handle == nullptr) { + break; + } + + if (rejectCxOverGlobalLimit()) { + // The global connection limit has been reached. + io_handle->close(); + cb_.onReject(); + continue; + } + + // Get the local address from the new socket if the listener is listening on IP ANY + // (e.g., 0.0.0.0 for IPv4) (local_address_ is nullptr in this case). + const Address::InstanceConstSharedPtr& local_address = + local_address_ ? local_address_ : io_handle->localAddress(); + + // The accept() call that filled in remote_addr doesn't fill in more than the sa_family field + // for Unix domain sockets; apparently there isn't a mechanism in the kernel to get the + // `sockaddr_un` associated with the client socket when starting from the server socket. + // We work around this by using our own name for the socket in this case. + // Pass the 'v6only' parameter as true if the local_address is an IPv6 address. This has no + // effect if the socket is a v4 socket, but for v6 sockets this will create an IPv4 remote + // address if an IPv4 local_address was created from an IPv6 mapped IPv4 address. + const Address::InstanceConstSharedPtr& remote_address = + (remote_addr.ss_family == AF_UNIX) + ? io_handle->peerAddress() + : Address::addressFromSockAddr(remote_addr, remote_addr_len, + local_address->ip()->version() == + Address::IpVersion::v6); + + cb_.onAccept( + std::make_unique(std::move(io_handle), local_address, remote_address)); } - - // Get the local address from the new socket if the listener is listening on IP ANY - // (e.g., 0.0.0.0 for IPv4) (local_address_ is nullptr in this case). - const Address::InstanceConstSharedPtr& local_address = - listener->local_address_ ? listener->local_address_ : io_handle->localAddress(); - - // The accept() call that filled in remote_addr doesn't fill in more than the sa_family field - // for Unix domain sockets; apparently there isn't a mechanism in the kernel to get the - // `sockaddr_un` associated with the client socket when starting from the server socket. - // We work around this by using our own name for the socket in this case. - // Pass the 'v6only' parameter as true if the local_address is an IPv6 address. This has no effect - // if the socket is a v4 socket, but for v6 sockets this will create an IPv4 remote address if an - // IPv4 local_address was created from an IPv6 mapped IPv4 address. - const Address::InstanceConstSharedPtr& remote_address = - (remote_addr->sa_family == AF_UNIX) - ? io_handle->peerAddress() - : Address::addressFromSockAddr(*reinterpret_cast(remote_addr), - remote_addr_len, - local_address->ip()->version() == Address::IpVersion::v6); - listener->cb_.onAccept( - std::make_unique(std::move(io_handle), local_address, remote_address)); } void ListenerImpl::setupServerSocket(Event::DispatcherImpl& dispatcher, Socket& socket) { - listener_.reset( - evconnlistener_new(&dispatcher.base(), listenCallback, this, 0, -1, socket.ioHandle().fd())); + // TODO(fcoras): make listen backlog configurable. For now use 128, which is what libevent + // defaults to for listeners configured with a negative (unspecified) backlog + socket.ioHandle().listen(128); - if (!listener_) { - throw CreateListenerException( - fmt::format("cannot listen on socket: {}", socket.localAddress()->asString())); - } + // Although onSocketEvent drains to completion, use level triggered mode to avoid potential + // loss of the trigger due to transient accept errors. + file_event_ = dispatcher.createFileEvent( + socket.ioHandle().fd(), [this](uint32_t events) -> void { onSocketEvent(events); }, + Event::FileTriggerType::Level, Event::FileReadyType::Read); if (!Network::Socket::applyOptions(socket.options(), socket, envoy::config::core::v3::SocketOption::STATE_LISTENING)) { throw CreateListenerException(fmt::format("cannot set post-listen socket option on socket: {}", socket.localAddress()->asString())); } - - evconnlistener_set_error_cb(listener_.get(), errorCallback); } ListenerImpl::ListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket, ListenerCallbacks& cb, bool bind_to_port) - : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb), listener_(nullptr) { + : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb) { if (bind_to_port) { setupServerSocket(dispatcher, *socket_); } } -void ListenerImpl::errorCallback(evconnlistener*, void*) { - // We should never get an error callback. This can happen if we run out of FDs or memory. In those - // cases just crash. - PANIC(fmt::format("listener accept failure: {}", errorDetails(errno))); -} - -void ListenerImpl::enable() { - if (listener_.get()) { - evconnlistener_enable(listener_.get()); - } -} +void ListenerImpl::enable() { file_event_->setEnabled(Event::FileReadyType::Read); } -void ListenerImpl::disable() { - if (listener_.get()) { - evconnlistener_disable(listener_.get()); - } -} +void ListenerImpl::disable() { file_event_->setEnabled(0); } } // namespace Network } // namespace Envoy diff --git a/source/common/network/listener_impl.h b/source/common/network/listener_impl.h index c431d77f4610..e26eafebe5d2 100644 --- a/source/common/network/listener_impl.h +++ b/source/common/network/listener_impl.h @@ -16,7 +16,6 @@ class ListenerImpl : public BaseListenerImpl { public: ListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket, ListenerCallbacks& cb, bool bind_to_port); - void disable() override; void enable() override; @@ -28,15 +27,13 @@ class ListenerImpl : public BaseListenerImpl { ListenerCallbacks& cb_; private: - static void listenCallback(evconnlistener*, evutil_socket_t fd, sockaddr* remote_addr, - int remote_addr_len, void* arg); - static void errorCallback(evconnlistener* listener, void* context); + void onSocketEvent(short flags); // Returns true if global connection limit has been reached and the accepted socket should be // rejected/closed. If the accepted socket is to be admitted, false is returned. static bool rejectCxOverGlobalLimit(); - Event::Libevent::ListenerPtr listener_; + Event::FileEventPtr file_event_; }; } // namespace Network diff --git a/source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h b/source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h index 84420816b917..d58d4a106044 100644 --- a/source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h +++ b/source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h @@ -68,6 +68,9 @@ class QuicIoHandleWrapper : public Network::IoHandle { return io_handle_.bind(address); } Api::SysCallIntResult listen(int backlog) override { return io_handle_.listen(backlog); } + Network::IoHandlePtr accept(struct sockaddr* addr, socklen_t* addrlen) override { + return io_handle_.accept(addr, addrlen); + } Api::SysCallIntResult connect(Network::Address::InstanceConstSharedPtr address) override { return io_handle_.connect(address); } diff --git a/test/mocks/network/io_handle.h b/test/mocks/network/io_handle.h index 0b27d262bebc..5bfdcc23ac45 100644 --- a/test/mocks/network/io_handle.h +++ b/test/mocks/network/io_handle.h @@ -32,6 +32,7 @@ class MockIoHandle : public IoHandle { MOCK_METHOD(bool, supportsUdpGro, (), (const)); MOCK_METHOD(Api::SysCallIntResult, bind, (Address::InstanceConstSharedPtr address)); MOCK_METHOD(Api::SysCallIntResult, listen, (int backlog)); + MOCK_METHOD(IoHandlePtr, accept, (struct sockaddr * addr, socklen_t* addrlen)); MOCK_METHOD(Api::SysCallIntResult, connect, (Address::InstanceConstSharedPtr address)); MOCK_METHOD(Api::SysCallIntResult, setOption, (int level, int optname, const void* optval, socklen_t optlen)); From f7fc2acb2b02448708ee0fa0a799787ee5aad2b9 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 12 Aug 2020 10:41:34 -0700 Subject: [PATCH 42/67] build: use bazel native yaml-cpp (#12607) Signed-off-by: Lizan Zhou --- bazel/foreign_cc/BUILD | 16 ---------------- bazel/repositories.bzl | 7 ++----- bazel/repository_locations.bzl | 2 +- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index c4a59ab20bda..09a59c586964 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -211,22 +211,6 @@ envoy_cmake_external( }), ) -envoy_cmake_external( - name = "yaml", - cache_entries = { - "YAML_CPP_BUILD_TESTS": "off", - "YAML_CPP_BUILD_TOOLS": "off", - "YAML_BUILD_SHARED_LIBS": "off", - "CMAKE_CXX_COMPILER_FORCED": "on", - "YAML_MSVC_SHARED_RT": "off", - }, - lib_source = "@com_github_jbeder_yaml_cpp//:all", - static_libraries = select({ - "//bazel:windows_x86_64": ["yaml-cpp.lib"], - "//conditions:default": ["libyaml-cpp.a"], - }), -) - envoy_cmake_external( name = "zlib", cache_entries = { diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 4e0293ef288b..ee59f5564a78 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -339,15 +339,12 @@ def _com_github_google_libprotobuf_mutator(): ) def _com_github_jbeder_yaml_cpp(): - location = _get_location("com_github_jbeder_yaml_cpp") - http_archive( + _repository_impl( name = "com_github_jbeder_yaml_cpp", - build_file_content = BUILD_ALL_CONTENT, - **location ) native.bind( name = "yaml_cpp", - actual = "@envoy//bazel/foreign_cc:yaml", + actual = "@com_github_jbeder_yaml_cpp//:yaml-cpp", ) def _com_github_libevent_libevent(): diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 11cf908a21c0..07cce0b5698e 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -265,7 +265,7 @@ DEPENDENCY_REPOSITORIES = dict( sha256 = "79ab7069ef1c7c3632e7ffe095f7185d4c77b64d8035db3c085c239d4fe96d5f", strip_prefix = "yaml-cpp-98acc5a8874faab28b82c28936f4b400b389f5d6", # 2020-07-28 - urls = ["https://github.com/greenhouse-org/yaml-cpp/archive/98acc5a8874faab28b82c28936f4b400b389f5d6.tar.gz"], + urls = ["https://github.com/jbeder/yaml-cpp/archive/98acc5a8874faab28b82c28936f4b400b389f5d6.tar.gz"], use_category = ["dataplane"], cpe = "N/A", ), From ea7665a077e6d97194b85a37e550e162df314696 Mon Sep 17 00:00:00 2001 From: Sotiris Nanopoulos Date: Wed, 12 Aug 2020 10:43:25 -0700 Subject: [PATCH 43/67] network: adds PlatformDefaultTriggerType for FileTriggerType (#12584) Adds PlatformDefaultTriggerType which is used to create file events. The main motivation for using this is the following: 1) Easier to swap out between FileTriggerType. This is useful for UNIX devs to debug issues that might affect Windows. 2) Removes various #ifdefs around the codebase. With this change, proxy_protocol_test now passes on Windows. Signed-off-by: Sotiris Nanopoulos --- include/envoy/event/file_event.h | 8 ++++++ source/common/event/libevent_scheduler.cc | 10 +------- source/common/event/libevent_scheduler.h | 11 ++++++++ source/common/network/connection_impl.cc | 8 ++---- source/common/network/udp_listener_impl.cc | 2 +- .../listener/http_inspector/http_inspector.cc | 3 ++- .../listener/proxy_protocol/proxy_protocol.cc | 2 +- .../listener/tls_inspector/tls_inspector.cc | 3 ++- .../filters/udp/udp_proxy/udp_proxy_filter.cc | 2 +- source/server/hot_restarting_parent.cc | 2 +- test/common/event/file_event_impl_test.cc | 25 ++++--------------- .../http_inspector/http_inspector_test.cc | 2 +- .../filters/listener/proxy_protocol/BUILD | 1 - .../tls_inspector/tls_inspector_test.cc | 4 +-- .../udp/udp_proxy/udp_proxy_filter_test.cc | 5 ++-- 15 files changed, 41 insertions(+), 47 deletions(-) diff --git a/include/envoy/event/file_event.h b/include/envoy/event/file_event.h index 72f618097ec5..e66289cdb4f0 100644 --- a/include/envoy/event/file_event.h +++ b/include/envoy/event/file_event.h @@ -20,6 +20,14 @@ struct FileReadyType { enum class FileTriggerType { Level, Edge }; +static constexpr FileTriggerType PlatformDefaultTriggerType +#ifdef WIN32 + // Libevent only supports Level trigger on Windows. + {FileTriggerType::Level}; +#else + {FileTriggerType::Edge}; +#endif + /** * Callback invoked when a FileEvent is ready for reading or writing. */ diff --git a/source/common/event/libevent_scheduler.cc b/source/common/event/libevent_scheduler.cc index dda0380cb4d8..e0f21c5da2ba 100644 --- a/source/common/event/libevent_scheduler.cc +++ b/source/common/event/libevent_scheduler.cc @@ -48,15 +48,7 @@ void LibeventScheduler::run(Dispatcher::RunType mode) { int flag = 0; switch (mode) { case Dispatcher::RunType::NonBlock: - flag = EVLOOP_NONBLOCK; -#ifdef WIN32 - // On Windows, EVLOOP_NONBLOCK will cause the libevent event_base_loop to run forever. - // This is because libevent only supports level triggering on Windows, and so the write - // event callbacks will trigger every time through the loop. Adding EVLOOP_ONCE ensures the - // loop will run at most once - flag |= EVLOOP_ONCE; -#endif - break; + flag = LibeventScheduler::flagsBasedOnEventType(); case Dispatcher::RunType::Block: // The default flags have 'block' behavior. See // http://www.wangafu.net/~nickm/libevent-book/Ref3_eventloop.html diff --git a/source/common/event/libevent_scheduler.h b/source/common/event/libevent_scheduler.h index 6059a0017bae..ab076cbabd88 100644 --- a/source/common/event/libevent_scheduler.h +++ b/source/common/event/libevent_scheduler.h @@ -108,6 +108,17 @@ class LibeventScheduler : public Scheduler, public CallbackScheduler { static void onPrepareForStats(evwatch*, const evwatch_prepare_cb_info* info, void* arg); static void onCheckForStats(evwatch*, const evwatch_check_cb_info*, void* arg); + static constexpr int flagsBasedOnEventType() { + if constexpr (Event::PlatformDefaultTriggerType == FileTriggerType::Level) { + // On Windows, EVLOOP_NONBLOCK will cause the libevent event_base_loop to run forever. + // This is because libevent only supports level triggering on Windows, and so the write + // event callbacks will trigger every time through the loop. Adding EVLOOP_ONCE ensures the + // loop will run at most once + return EVLOOP_NONBLOCK | EVLOOP_ONCE; + } + return EVLOOP_NONBLOCK; + } + Libevent::BasePtr libevent_; DispatcherStats* stats_{}; // stats owned by the containing DispatcherImpl bool timeout_set_{}; // whether there is a poll timeout in the current event loop iteration diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index 2abbea352b76..5e542caae0f3 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -65,12 +65,8 @@ ConnectionImpl::ConnectionImpl(Event::Dispatcher& dispatcher, ConnectionSocketPt connecting_ = true; } - // Libevent only supports Level trigger on Windows. -#ifdef WIN32 - Event::FileTriggerType trigger = Event::FileTriggerType::Level; -#else - Event::FileTriggerType trigger = Event::FileTriggerType::Edge; -#endif + Event::FileTriggerType trigger = Event::PlatformDefaultTriggerType; + // We never ask for both early close and read at the same time. If we are reading, we want to // consume all available data. file_event_ = dispatcher_.createFileEvent( diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index 3eaf0272e940..84d42555f1b4 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -33,7 +33,7 @@ UdpListenerImpl::UdpListenerImpl(Event::DispatcherImpl& dispatcher, SocketShared : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb), time_source_(time_source) { file_event_ = dispatcher_.createFileEvent( socket_->ioHandle().fd(), [this](uint32_t events) -> void { onSocketEvent(events); }, - Event::FileTriggerType::Edge, Event::FileReadyType::Read | Event::FileReadyType::Write); + Event::PlatformDefaultTriggerType, Event::FileReadyType::Read | Event::FileReadyType::Write); ASSERT(file_event_); diff --git a/source/extensions/filters/listener/http_inspector/http_inspector.cc b/source/extensions/filters/listener/http_inspector/http_inspector.cc index 90234d9b31fe..3a7f46addb47 100644 --- a/source/extensions/filters/listener/http_inspector/http_inspector.cc +++ b/source/extensions/filters/listener/http_inspector/http_inspector.cc @@ -93,7 +93,8 @@ Network::FilterStatus Filter::onAccept(Network::ListenerFilterCallbacks& cb) { break; } }, - Event::FileTriggerType::Edge, Event::FileReadyType::Read | Event::FileReadyType::Closed); + Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read | Event::FileReadyType::Closed); return Network::FilterStatus::StopIteration; } NOT_REACHED_GCOVR_EXCL_LINE; diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index c3029c2234cf..9c474bc8b420 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -74,7 +74,7 @@ Network::FilterStatus Filter::onAccept(Network::ListenerFilterCallbacks& cb) { ASSERT(events == Event::FileReadyType::Read); onRead(); }, - Event::FileTriggerType::Edge, Event::FileReadyType::Read); + Event::PlatformDefaultTriggerType, Event::FileReadyType::Read); cb_ = &cb; return Network::FilterStatus::StopIteration; } diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc index 38ea9324b243..87095b65c970 100644 --- a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc +++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc @@ -115,7 +115,8 @@ Network::FilterStatus Filter::onAccept(Network::ListenerFilterCallbacks& cb) { break; } }, - Event::FileTriggerType::Edge, Event::FileReadyType::Read | Event::FileReadyType::Closed); + Event::PlatformDefaultTriggerType, + Event::FileReadyType::Read | Event::FileReadyType::Closed); return Network::FilterStatus::StopIteration; } NOT_REACHED_GCOVR_EXCL_LINE; diff --git a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc index 095bc869f7e6..766e3dac4873 100644 --- a/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc +++ b/source/extensions/filters/udp/udp_proxy/udp_proxy_filter.cc @@ -159,7 +159,7 @@ UdpProxyFilter::ActiveSession::ActiveSession(ClusterInfo& cluster, // is bound until the first packet is sent to the upstream host. io_handle_(cluster.filter_.createIoHandle(host)), socket_event_(cluster.filter_.read_callbacks_->udpListener().dispatcher().createFileEvent( - io_handle_->fd(), [this](uint32_t) { onReadReady(); }, Event::FileTriggerType::Edge, + io_handle_->fd(), [this](uint32_t) { onReadReady(); }, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read)) { ENVOY_LOG(debug, "creating new session: downstream={} local={} upstream={}", addresses_.peer_->asStringView(), addresses_.local_->asStringView(), diff --git a/source/server/hot_restarting_parent.cc b/source/server/hot_restarting_parent.cc index 5049c8077f91..c9d28bd1549b 100644 --- a/source/server/hot_restarting_parent.cc +++ b/source/server/hot_restarting_parent.cc @@ -28,7 +28,7 @@ void HotRestartingParent::initialize(Event::Dispatcher& dispatcher, Server::Inst ASSERT(events == Event::FileReadyType::Read); onSocketEvent(); }, - Event::FileTriggerType::Edge, Event::FileReadyType::Read); + Event::PlatformDefaultTriggerType, Event::FileReadyType::Read); internal_ = std::make_unique(&server); } diff --git a/test/common/event/file_event_impl_test.cc b/test/common/event/file_event_impl_test.cc index ca34b5eaf844..7116fd8bc5f7 100644 --- a/test/common/event/file_event_impl_test.cc +++ b/test/common/event/file_event_impl_test.cc @@ -89,11 +89,7 @@ TEST_P(FileEventImplActivateTest, Activate) { ReadyWatcher closed_event; EXPECT_CALL(closed_event, ready()).Times(1); -#ifdef WIN32 - const FileTriggerType trigger = FileTriggerType::Level; -#else - const FileTriggerType trigger = FileTriggerType::Edge; -#endif + const FileTriggerType trigger = Event::PlatformDefaultTriggerType; Event::FileEventPtr file_event = dispatcher->createFileEvent( fd, @@ -133,11 +129,7 @@ TEST_P(FileEventImplActivateTest, ActivateChaining) { evwatch_prepare_new(&static_cast(dispatcher.get())->base(), onWatcherReady, &prepare_watcher); -#ifdef WIN32 - const FileTriggerType trigger = FileTriggerType::Level; -#else - const FileTriggerType trigger = FileTriggerType::Edge; -#endif + const FileTriggerType trigger = Event::PlatformDefaultTriggerType; Event::FileEventPtr file_event = dispatcher->createFileEvent( fd, @@ -213,11 +205,7 @@ TEST_P(FileEventImplActivateTest, SetEnableCancelsActivate) { evwatch_prepare_new(&static_cast(dispatcher.get())->base(), onWatcherReady, &prepare_watcher); -#ifdef WIN32 - const FileTriggerType trigger = FileTriggerType::Level; -#else - const FileTriggerType trigger = FileTriggerType::Edge; -#endif + const FileTriggerType trigger = Event::PlatformDefaultTriggerType; Event::FileEventPtr file_event = dispatcher->createFileEvent( fd, @@ -318,11 +306,8 @@ TEST_F(FileEventImplTest, SetEnabled) { ReadyWatcher write_event; EXPECT_CALL(write_event, ready()).Times(2); -#ifdef WIN32 - const FileTriggerType trigger = FileTriggerType::Level; -#else - const FileTriggerType trigger = FileTriggerType::Edge; -#endif + const FileTriggerType trigger = Event::PlatformDefaultTriggerType; + Event::FileEventPtr file_event = dispatcher_->createFileEvent( fds_[0], [&](uint32_t events) -> void { diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc index a6638892f26f..c292f9dcd0bd 100644 --- a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc +++ b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc @@ -47,7 +47,7 @@ class HttpInspectorTest : public testing::Test { .WillOnce(Return(Api::SysCallSizeResult{static_cast(0), 0})); EXPECT_CALL(dispatcher_, - createFileEvent_(_, _, Event::FileTriggerType::Edge, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read | Event::FileReadyType::Closed)) .WillOnce(DoAll(SaveArg<1>(&file_event_callback_), ReturnNew>())); diff --git a/test/extensions/filters/listener/proxy_protocol/BUILD b/test/extensions/filters/listener/proxy_protocol/BUILD index f37389795778..fa746dbaa5c6 100644 --- a/test/extensions/filters/listener/proxy_protocol/BUILD +++ b/test/extensions/filters/listener/proxy_protocol/BUILD @@ -15,7 +15,6 @@ envoy_extension_cc_test( name = "proxy_protocol_test", srcs = ["proxy_protocol_test.cc"], extension_name = "envoy.filters.listener.proxy_protocol", - tags = ["fails_on_windows"], deps = [ "//source/common/buffer:buffer_lib", "//source/common/event:dispatcher_includes", diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc index 56f2e637e0fd..82f9d4811266 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc @@ -50,7 +50,7 @@ class TlsInspectorTest : public testing::TestWithParam(0), 0}; })); EXPECT_CALL(dispatcher_, - createFileEvent_(_, _, Event::FileTriggerType::Edge, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read | Event::FileReadyType::Closed)) .WillOnce( DoAll(SaveArg<1>(&file_event_callback_), ReturnNew>())); @@ -273,7 +273,7 @@ TEST_P(TlsInspectorTest, InlineReadSucceed) { // No event is created if the inline recv parse the hello. EXPECT_CALL(dispatcher_, - createFileEvent_(_, _, Event::FileTriggerType::Edge, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read | Event::FileReadyType::Closed)) .Times(0); diff --git a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc index 449a98c0a5cc..dcaf3846fb00 100644 --- a/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc +++ b/test/extensions/filters/udp/udp_proxy/udp_proxy_filter_test.cc @@ -163,8 +163,9 @@ class UdpProxyFilterTest : public testing::Test { EXPECT_CALL(*filter_, createIoHandle(_)) .WillOnce(Return(ByMove(Network::IoHandlePtr{test_sessions_.back().io_handle_}))); EXPECT_CALL(*new_session.io_handle_, fd()); - EXPECT_CALL(callbacks_.udp_listener_.dispatcher_, - createFileEvent_(_, _, Event::FileTriggerType::Edge, Event::FileReadyType::Read)) + EXPECT_CALL( + callbacks_.udp_listener_.dispatcher_, + createFileEvent_(_, _, Event::PlatformDefaultTriggerType, Event::FileReadyType::Read)) .WillOnce(DoAll(SaveArg<1>(&new_session.file_event_cb_), Return(nullptr))); // Internal Buffer is Empty, flush will be a no-op ON_CALL(callbacks_.udp_listener_, flush()) From a7e7bd1b3f6e508ad1f85baf4486a942a3455a0d Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Wed, 12 Aug 2020 10:44:30 -0700 Subject: [PATCH 44/67] caching: Refactored registration of inline headers in the CacheFilter tree to a separate header file (#12576) Refactored registration of inline headers in the CacheFilter tree to a separate header file. Signed-off-by: Yosry Ahmed --- source/extensions/filters/http/cache/BUILD | 12 ++++- .../filters/http/cache/cache_filter.cc | 17 ++++--- .../filters/http/cache/cache_filter.h | 1 - .../filters/http/cache/cacheability_utils.cc | 9 ++-- .../filters/http/cache/http_cache.cc | 11 ++--- .../http/cache/inline_headers_handles.h | 47 +++++++++++++++++++ 6 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 source/extensions/filters/http/cache/inline_headers_handles.h diff --git a/source/extensions/filters/http/cache/BUILD b/source/extensions/filters/http/cache/BUILD index 6dd67613de95..0b774d70b755 100644 --- a/source/extensions/filters/http/cache/BUILD +++ b/source/extensions/filters/http/cache/BUILD @@ -20,10 +20,10 @@ envoy_cc_library( ":cache_headers_utils_lib", ":cacheability_utils_lib", ":http_cache_lib", + ":inline_headers_handles", "//source/common/common:enum_to_int", "//source/common/common:logger_lib", "//source/common/common:macros", - "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", "//source/extensions/filters/http/common:pass_through_filter_lib", @@ -37,6 +37,7 @@ envoy_cc_library( hdrs = ["cacheability_utils.h"], deps = [ ":cache_headers_utils_lib", + ":inline_headers_handles", "//source/common/common:utility_lib", "//source/common/http:headers_lib", ], @@ -53,6 +54,7 @@ envoy_cc_library( hdrs = ["http_cache.h"], deps = [ ":cache_headers_utils_lib", + ":inline_headers_handles", ":key_cc_proto", "//include/envoy/buffer:buffer_interface", "//include/envoy/common:time_interface", @@ -78,6 +80,14 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "inline_headers_handles", + hdrs = ["inline_headers_handles.h"], + deps = [ + "//source/common/http:headers_lib", + ], +) + envoy_cc_extension( name = "config", srcs = ["config.cc"], diff --git a/source/extensions/filters/http/cache/cache_filter.cc b/source/extensions/filters/http/cache/cache_filter.cc index 6e4b469f342e..18ab3e625c26 100644 --- a/source/extensions/filters/http/cache/cache_filter.cc +++ b/source/extensions/filters/http/cache/cache_filter.cc @@ -5,6 +5,7 @@ #include "common/http/utility.h" #include "extensions/filters/http/cache/cacheability_utils.h" +#include "extensions/filters/http/cache/inline_headers_handles.h" #include "absl/strings/string_view.h" @@ -274,9 +275,8 @@ bool CacheFilter::shouldUpdateCachedEntry(const Http::ResponseHeaderMap& respons // and assuming a single cached response per key: // If the 304 response contains a strong validator (etag) that does not match the cached response, // the cached response should not be updated. - const Http::HeaderEntry* response_etag = response_headers.get(Http::CustomHeaders::get().Etag); - const Http::HeaderEntry* cached_etag = - lookup_result_->headers_->get(Http::CustomHeaders::get().Etag); + const Http::HeaderEntry* response_etag = response_headers.getInline(etag_handle.handle()); + const Http::HeaderEntry* cached_etag = lookup_result_->headers_->getInline(etag_handle.handle()); return !response_etag || (cached_etag && cached_etag->value().getStringView() == response_etag->value().getStringView()); } @@ -288,25 +288,24 @@ void CacheFilter::injectValidationHeaders(Http::RequestHeaderMap& request_header "injectValidationHeaders precondition unsatisfied: the " "CacheFilter is not validating a cache lookup result"); - const Http::HeaderEntry* etag_header = - lookup_result_->headers_->get(Http::CustomHeaders::get().Etag); + const Http::HeaderEntry* etag_header = lookup_result_->headers_->getInline(etag_handle.handle()); const Http::HeaderEntry* last_modified_header = - lookup_result_->headers_->get(Http::CustomHeaders::get().LastModified); + lookup_result_->headers_->getInline(last_modified_handle.handle()); if (etag_header) { absl::string_view etag = etag_header->value().getStringView(); - request_headers.setReferenceKey(Http::CustomHeaders::get().IfNoneMatch, etag); + request_headers.setInline(if_none_match_handle.handle(), etag); } if (CacheHeadersUtils::httpTime(last_modified_header) != SystemTime()) { // Valid Last-Modified header exists. absl::string_view last_modified = last_modified_header->value().getStringView(); - request_headers.setReferenceKey(Http::CustomHeaders::get().IfModifiedSince, last_modified); + request_headers.setInline(if_modified_since_handle.handle(), last_modified); } else { // Either Last-Modified is missing or invalid, fallback to Date. // A correct behaviour according to: // https://httpwg.org/specs/rfc7232.html#header.if-modified-since absl::string_view date = lookup_result_->headers_->getDateValue(); - request_headers.setReferenceKey(Http::CustomHeaders::get().IfModifiedSince, date); + request_headers.setInline(if_modified_since_handle.handle(), date); } } diff --git a/source/extensions/filters/http/cache/cache_filter.h b/source/extensions/filters/http/cache/cache_filter.h index f873569289e0..97f37d827671 100644 --- a/source/extensions/filters/http/cache/cache_filter.h +++ b/source/extensions/filters/http/cache/cache_filter.h @@ -6,7 +6,6 @@ #include #include "envoy/extensions/filters/http/cache/v3alpha/cache.pb.h" -#include "envoy/http/header_map.h" #include "common/common/logger.h" diff --git a/source/extensions/filters/http/cache/cacheability_utils.cc b/source/extensions/filters/http/cache/cacheability_utils.cc index 778fd574a09d..784c37953c9d 100644 --- a/source/extensions/filters/http/cache/cacheability_utils.cc +++ b/source/extensions/filters/http/cache/cacheability_utils.cc @@ -5,6 +5,8 @@ #include "common/common/macros.h" #include "common/common/utility.h" +#include "extensions/filters/http/cache/inline_headers_handles.h" + namespace Envoy { namespace Extensions { namespace HttpFilters { @@ -30,11 +32,6 @@ const std::vector& conditionalHeaders() { } } // namespace -Http::RegisterCustomInlineHeader - authorization_handle(Http::CustomHeaders::get().Authorization); -Http::RegisterCustomInlineHeader - cache_control_handle(Http::CustomHeaders::get().CacheControl); - bool CacheabilityUtils::isCacheableRequest(const Http::RequestHeaderMap& headers) { const absl::string_view method = headers.getMethodValue(); const absl::string_view forwarded_proto = headers.getForwardedProtoValue(); @@ -60,7 +57,7 @@ bool CacheabilityUtils::isCacheableRequest(const Http::RequestHeaderMap& headers } bool CacheabilityUtils::isCacheableResponse(const Http::ResponseHeaderMap& headers) { - absl::string_view cache_control = headers.getInlineValue(cache_control_handle.handle()); + absl::string_view cache_control = headers.getInlineValue(response_cache_control_handle.handle()); ResponseCacheControl response_cache_control(cache_control); // Only cache responses with explicit validation data, either: diff --git a/source/extensions/filters/http/cache/http_cache.cc b/source/extensions/filters/http/cache/http_cache.cc index 60e73ff89549..43943444c897 100644 --- a/source/extensions/filters/http/cache/http_cache.cc +++ b/source/extensions/filters/http/cache/http_cache.cc @@ -11,6 +11,8 @@ #include "common/http/headers.h" #include "common/protobuf/utility.h" +#include "extensions/filters/http/cache/inline_headers_handles.h" + #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/time/time.h" @@ -20,13 +22,6 @@ namespace Extensions { namespace HttpFilters { namespace Cache { -Http::RegisterCustomInlineHeader - request_cache_control_handle(Http::CustomHeaders::get().CacheControl); -Http::RegisterCustomInlineHeader - response_cache_control_handle(Http::CustomHeaders::get().CacheControl); -Http::RegisterCustomInlineHeader - pragma_handler(Http::CustomHeaders::get().Pragma); - std::ostream& operator<<(std::ostream& os, CacheEntryStatus status) { switch (status) { case CacheEntryStatus::Ok: @@ -90,7 +85,7 @@ size_t localHashKey(const Key& key) { return stableHashKey(key); } void LookupRequest::initializeRequestCacheControl(const Http::RequestHeaderMap& request_headers) { const absl::string_view cache_control = request_headers.getInlineValue(request_cache_control_handle.handle()); - const absl::string_view pragma = request_headers.getInlineValue(pragma_handler.handle()); + const absl::string_view pragma = request_headers.getInlineValue(pragma_handle.handle()); if (!cache_control.empty()) { request_cache_control_ = RequestCacheControl(cache_control); diff --git a/source/extensions/filters/http/cache/inline_headers_handles.h b/source/extensions/filters/http/cache/inline_headers_handles.h new file mode 100644 index 000000000000..c9ae8353a8e2 --- /dev/null +++ b/source/extensions/filters/http/cache/inline_headers_handles.h @@ -0,0 +1,47 @@ +#pragma once + +#include "common/http/headers.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Cache { + +inline Http::RegisterCustomInlineHeader + authorization_handle(Http::CustomHeaders::get().Authorization); + +inline Http::RegisterCustomInlineHeader + pragma_handle(Http::CustomHeaders::get().Pragma); + +inline Http::RegisterCustomInlineHeader + request_cache_control_handle(Http::CustomHeaders::get().CacheControl); + +inline Http::RegisterCustomInlineHeader + if_match_handle(Http::CustomHeaders::get().IfMatch); + +inline Http::RegisterCustomInlineHeader + if_none_match_handle(Http::CustomHeaders::get().IfNoneMatch); + +inline Http::RegisterCustomInlineHeader + if_modified_since_handle(Http::CustomHeaders::get().IfModifiedSince); + +inline Http::RegisterCustomInlineHeader + if_unmodified_since_handle(Http::CustomHeaders::get().IfUnmodifiedSince); + +inline Http::RegisterCustomInlineHeader + if_range_handle(Http::CustomHeaders::get().IfRange); + +// Response headers inline handles +inline Http::RegisterCustomInlineHeader + response_cache_control_handle(Http::CustomHeaders::get().CacheControl); + +inline Http::RegisterCustomInlineHeader + last_modified_handle(Http::CustomHeaders::get().LastModified); + +inline Http::RegisterCustomInlineHeader + etag_handle(Http::CustomHeaders::get().Etag); + +} // namespace Cache +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy From 18a51245bb5460ed177ca5e4cf32acbf97b116be Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 12 Aug 2020 10:46:51 -0700 Subject: [PATCH 45/67] network: address socket interface pointer instead of name (#12550) Signed-off-by: Florin Coras --- include/envoy/network/address.h | 8 +++- source/common/network/address_impl.cc | 44 ++++++++++--------- source/common/network/address_impl.h | 36 ++++++++------- source/common/network/socket_interface.h | 9 +--- source/common/network/utility.cc | 5 ++- test/common/network/connection_impl_test.cc | 12 ++--- test/common/network/dns_impl_test.cc | 5 ++- test/common/tracing/http_tracer_impl_test.cc | 32 +++++++------- .../http_connection_manager/config_test.cc | 4 +- .../transport_sockets/tls/ssl_socket_test.cc | 4 +- .../socket_interface_integration_test.cc | 5 ++- test/mocks/network/mocks.h | 6 ++- test/server/connection_handler_test.cc | 2 +- 13 files changed, 91 insertions(+), 81 deletions(-) diff --git a/include/envoy/network/address.h b/include/envoy/network/address.h index 94793f12c155..edea7108a7b9 100644 --- a/include/envoy/network/address.h +++ b/include/envoy/network/address.h @@ -16,6 +16,10 @@ namespace Envoy { namespace Network { + +/* Forward declaration */ +class SocketInterface; + namespace Address { /** @@ -178,9 +182,9 @@ class Instance { virtual Type type() const PURE; /** - * @return name of socket interface that should be used with this address + * @return SocketInterface to be used with the address */ - virtual const std::string& socketInterface() const PURE; + virtual const Network::SocketInterface& socketInterface() const PURE; }; using InstanceConstSharedPtr = std::shared_ptr; diff --git a/source/common/network/address_impl.cc b/source/common/network/address_impl.cc index 57d1317b7e4d..2fb068d94674 100644 --- a/source/common/network/address_impl.cc +++ b/source/common/network/address_impl.cc @@ -45,6 +45,10 @@ std::string friendlyNameFromAbstractPath(absl::string_view path) { return friendly_name; } +const SocketInterface* sockInterfaceOrDefault(const SocketInterface* sock_interface) { + return sock_interface == nullptr ? &SocketInterfaceSingleton::get() : sock_interface; +} + } // namespace Address::InstanceConstSharedPtr addressFromSockAddr(const sockaddr_storage& ss, socklen_t ss_len, @@ -91,8 +95,8 @@ Address::InstanceConstSharedPtr addressFromSockAddr(const sockaddr_storage& ss, NOT_REACHED_GCOVR_EXCL_LINE; } -Ipv4Instance::Ipv4Instance(const sockaddr_in* address, absl::string_view sock_interface) - : InstanceBase(Type::Ip, sock_interface) { +Ipv4Instance::Ipv4Instance(const sockaddr_in* address, const SocketInterface* sock_interface) + : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { memset(&ip_.ipv4_.address_, 0, sizeof(ip_.ipv4_.address_)); ip_.ipv4_.address_ = *address; ip_.friendly_address_ = sockaddrToString(*address); @@ -106,12 +110,12 @@ Ipv4Instance::Ipv4Instance(const sockaddr_in* address, absl::string_view sock_in validateIpv4Supported(friendly_name_); } -Ipv4Instance::Ipv4Instance(const std::string& address, absl::string_view sock_interface) - : Ipv4Instance(address, 0, sock_interface) {} +Ipv4Instance::Ipv4Instance(const std::string& address, const SocketInterface* sock_interface) + : Ipv4Instance(address, 0, sockInterfaceOrDefault(sock_interface)) {} Ipv4Instance::Ipv4Instance(const std::string& address, uint32_t port, - absl::string_view sock_interface) - : InstanceBase(Type::Ip, sock_interface) { + const SocketInterface* sock_interface) + : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { memset(&ip_.ipv4_.address_, 0, sizeof(ip_.ipv4_.address_)); ip_.ipv4_.address_.sin_family = AF_INET; ip_.ipv4_.address_.sin_port = htons(port); @@ -125,8 +129,8 @@ Ipv4Instance::Ipv4Instance(const std::string& address, uint32_t port, ip_.friendly_address_ = address; } -Ipv4Instance::Ipv4Instance(uint32_t port, absl::string_view sock_interface) - : InstanceBase(Type::Ip, sock_interface) { +Ipv4Instance::Ipv4Instance(uint32_t port, const SocketInterface* sock_interface) + : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { memset(&ip_.ipv4_.address_, 0, sizeof(ip_.ipv4_.address_)); ip_.ipv4_.address_.sin_family = AF_INET; ip_.ipv4_.address_.sin_port = htons(port); @@ -188,8 +192,8 @@ std::string Ipv6Instance::Ipv6Helper::makeFriendlyAddress() const { } Ipv6Instance::Ipv6Instance(const sockaddr_in6& address, bool v6only, - absl::string_view sock_interface) - : InstanceBase(Type::Ip, sock_interface) { + const SocketInterface* sock_interface) + : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { ip_.ipv6_.address_ = address; ip_.friendly_address_ = ip_.ipv6_.makeFriendlyAddress(); ip_.ipv6_.v6only_ = v6only; @@ -197,12 +201,12 @@ Ipv6Instance::Ipv6Instance(const sockaddr_in6& address, bool v6only, validateIpv6Supported(friendly_name_); } -Ipv6Instance::Ipv6Instance(const std::string& address, absl::string_view sock_interface) - : Ipv6Instance(address, 0, sock_interface) {} +Ipv6Instance::Ipv6Instance(const std::string& address, const SocketInterface* sock_interface) + : Ipv6Instance(address, 0, sockInterfaceOrDefault(sock_interface)) {} Ipv6Instance::Ipv6Instance(const std::string& address, uint32_t port, - absl::string_view sock_interface) - : InstanceBase(Type::Ip, sock_interface) { + const SocketInterface* sock_interface) + : InstanceBase(Type::Ip, sockInterfaceOrDefault(sock_interface)) { ip_.ipv6_.address_.sin6_family = AF_INET6; ip_.ipv6_.address_.sin6_port = htons(port); if (!address.empty()) { @@ -218,8 +222,8 @@ Ipv6Instance::Ipv6Instance(const std::string& address, uint32_t port, validateIpv6Supported(friendly_name_); } -Ipv6Instance::Ipv6Instance(uint32_t port, absl::string_view sock_interface) - : Ipv6Instance("", port, sock_interface) {} +Ipv6Instance::Ipv6Instance(uint32_t port, const SocketInterface* sock_interface) + : Ipv6Instance("", port, sockInterfaceOrDefault(sock_interface)) {} bool Ipv6Instance::operator==(const Instance& rhs) const { const auto* rhs_casted = dynamic_cast(&rhs); @@ -228,8 +232,8 @@ bool Ipv6Instance::operator==(const Instance& rhs) const { } PipeInstance::PipeInstance(const sockaddr_un* address, socklen_t ss_len, mode_t mode, - absl::string_view sock_interface) - : InstanceBase(Type::Pipe, sock_interface) { + const SocketInterface* sock_interface) + : InstanceBase(Type::Pipe, sockInterfaceOrDefault(sock_interface)) { if (address->sun_path[0] == '\0') { #if !defined(__linux__) throw EnvoyException("Abstract AF_UNIX sockets are only supported on linux."); @@ -254,8 +258,8 @@ PipeInstance::PipeInstance(const sockaddr_un* address, socklen_t ss_len, mode_t } PipeInstance::PipeInstance(const std::string& pipe_path, mode_t mode, - absl::string_view sock_interface) - : InstanceBase(Type::Pipe, sock_interface) { + const SocketInterface* sock_interface) + : InstanceBase(Type::Pipe, sockInterfaceOrDefault(sock_interface)) { if (pipe_path.size() >= sizeof(pipe_.address_.sun_path)) { throw EnvoyException( fmt::format("Path \"{}\" exceeds maximum UNIX domain socket path size of {}.", pipe_path, diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index c7473fcd4754..3b3ffd52783f 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -8,6 +8,7 @@ #include "envoy/common/platform.h" #include "envoy/network/address.h" +#include "envoy/network/socket.h" namespace Envoy { namespace Network { @@ -37,16 +38,14 @@ class InstanceBase : public Instance { const std::string& logicalName() const override { return asString(); } Type type() const override { return type_; } - const std::string& socketInterface() const override { return socket_interface_; } + const SocketInterface& socketInterface() const override { return socket_interface_; } protected: - InstanceBase(Type type) : type_(type) {} - InstanceBase(Type type, absl::string_view sock_interface) : type_(type) { - socket_interface_ = std::string(sock_interface); - } + InstanceBase(Type type, const SocketInterface* sock_interface) + : socket_interface_(*sock_interface), type_(type) {} std::string friendly_name_; - std::string socket_interface_; + const SocketInterface& socket_interface_; private: const Type type_; @@ -60,23 +59,26 @@ class Ipv4Instance : public InstanceBase { /** * Construct from an existing unix IPv4 socket address (IP v4 address and port). */ - explicit Ipv4Instance(const sockaddr_in* address, absl::string_view sock_interface = ""); + explicit Ipv4Instance(const sockaddr_in* address, + const SocketInterface* sock_interface = nullptr); /** * Construct from a string IPv4 address such as "1.2.3.4". Port will be unset/0. */ - explicit Ipv4Instance(const std::string& address, absl::string_view sock_interface = ""); + explicit Ipv4Instance(const std::string& address, + const SocketInterface* sock_interface = nullptr); /** * Construct from a string IPv4 address such as "1.2.3.4" as well as a port. */ - Ipv4Instance(const std::string& address, uint32_t port, absl::string_view sock_interface = ""); + Ipv4Instance(const std::string& address, uint32_t port, + const SocketInterface* sock_interface = nullptr); /** * Construct from a port. The IPv4 address will be set to "any" and is suitable for binding * a port to any available address. */ - explicit Ipv4Instance(uint32_t port, absl::string_view sock_interface = ""); + explicit Ipv4Instance(uint32_t port, const SocketInterface* sock_interface = nullptr); // Network::Address::Instance bool operator==(const Instance& rhs) const override; @@ -131,23 +133,25 @@ class Ipv6Instance : public InstanceBase { * Construct from an existing unix IPv6 socket address (IP v6 address and port). */ Ipv6Instance(const sockaddr_in6& address, bool v6only = true, - absl::string_view sock_interface = ""); + const SocketInterface* sock_interface = nullptr); /** * Construct from a string IPv6 address such as "12:34::5". Port will be unset/0. */ - explicit Ipv6Instance(const std::string& address, absl::string_view sock_interface = ""); + explicit Ipv6Instance(const std::string& address, + const SocketInterface* sock_interface = nullptr); /** * Construct from a string IPv6 address such as "12:34::5" as well as a port. */ - Ipv6Instance(const std::string& address, uint32_t port, absl::string_view sock_interface = ""); + Ipv6Instance(const std::string& address, uint32_t port, + const SocketInterface* sock_interface = nullptr); /** * Construct from a port. The IPv6 address will be set to "any" and is suitable for binding * a port to any available address. */ - explicit Ipv6Instance(uint32_t port, absl::string_view sock_interface = ""); + explicit Ipv6Instance(uint32_t port, const SocketInterface* sock_interface = nullptr); // Network::Address::Instance bool operator==(const Instance& rhs) const override; @@ -203,13 +207,13 @@ class PipeInstance : public InstanceBase { * Construct from an existing unix address. */ explicit PipeInstance(const sockaddr_un* address, socklen_t ss_len, mode_t mode = 0, - absl::string_view sock_interface = ""); + const SocketInterface* sock_interface = nullptr); /** * Construct from a string pipe path. */ explicit PipeInstance(const std::string& pipe_path, mode_t mode = 0, - absl::string_view sock_interface = ""); + const SocketInterface* sock_interface = nullptr); // Network::Address::Instance bool operator==(const Instance& rhs) const override; diff --git a/source/common/network/socket_interface.h b/source/common/network/socket_interface.h index 9374b65a2344..1a2b08a66481 100644 --- a/source/common/network/socket_interface.h +++ b/source/common/network/socket_interface.h @@ -60,14 +60,7 @@ using SocketInterfaceLoader = ScopedInjectableLoader; */ static inline IoHandlePtr ioHandleForAddr(Socket::Type type, const Address::InstanceConstSharedPtr addr) { - auto sock_interface_name = addr->socketInterface(); - if (!sock_interface_name.empty()) { - auto sock_interface = socketInterface(sock_interface_name); - RELEASE_ASSERT(sock_interface != nullptr, - fmt::format("missing socket interface {}", sock_interface_name)); - return sock_interface->socket(type, addr); - } - return SocketInterfaceSingleton::get().socket(type, addr); + return addr->socketInterface().socket(type, addr); } } // namespace Network diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 15145ec7ef49..158bdc79e20f 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -331,11 +331,12 @@ bool Utility::isLoopbackAddress(const Address::Instance& address) { Address::InstanceConstSharedPtr Utility::getCanonicalIpv4LoopbackAddress() { CONSTRUCT_ON_FIRST_USE(Address::InstanceConstSharedPtr, - new Address::Ipv4Instance("127.0.0.1", 0)); + new Address::Ipv4Instance("127.0.0.1", 0, nullptr)); } Address::InstanceConstSharedPtr Utility::getIpv6LoopbackAddress() { - CONSTRUCT_ON_FIRST_USE(Address::InstanceConstSharedPtr, new Address::Ipv6Instance("::1", 0)); + CONSTRUCT_ON_FIRST_USE(Address::InstanceConstSharedPtr, + new Address::Ipv6Instance("::1", 0, nullptr)); } Address::InstanceConstSharedPtr Utility::getIpv4AnyAddress() { diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index c4ea2f60c4fd..0a86156963a0 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -1079,11 +1079,11 @@ TEST_P(ConnectionImplTest, BindTest) { std::string address_string = TestUtility::getIpv4Loopback(); if (GetParam() == Network::Address::IpVersion::v4) { source_address_ = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance(address_string, 0)}; + new Network::Address::Ipv4Instance(address_string, 0, nullptr)}; } else { address_string = "::1"; source_address_ = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv6Instance(address_string, 0)}; + new Network::Address::Ipv6Instance(address_string, 0, nullptr)}; } setUpBasicConnection(); connect(); @@ -1097,11 +1097,11 @@ TEST_P(ConnectionImplTest, BindFromSocketTest) { Address::InstanceConstSharedPtr new_source_address; if (GetParam() == Network::Address::IpVersion::v4) { new_source_address = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance(address_string, 0)}; + new Network::Address::Ipv4Instance(address_string, 0, nullptr)}; } else { address_string = "::1"; new_source_address = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv6Instance(address_string, 0)}; + new Network::Address::Ipv6Instance(address_string, 0, nullptr)}; } auto option = std::make_shared>(); EXPECT_CALL(*option, setOption(_, Eq(envoy::config::core::v3::SocketOption::STATE_PREBIND))) @@ -1123,11 +1123,11 @@ TEST_P(ConnectionImplTest, BindFailureTest) { if (GetParam() == Network::Address::IpVersion::v6) { const std::string address_string = TestUtility::getIpv4Loopback(); source_address_ = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance(address_string, 0)}; + new Network::Address::Ipv4Instance(address_string, 0, nullptr)}; } else { const std::string address_string = "::1"; source_address_ = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv6Instance(address_string, 0)}; + new Network::Address::Ipv6Instance(address_string, 0, nullptr)}; } dispatcher_ = api_->allocateDispatcher("test_thread"); socket_ = std::make_shared( diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index df6aed9816bb..8b49d8035bca 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -393,12 +393,13 @@ class CustomInstance : public Address::Instance { const sockaddr* sockAddr() const override { return instance_.sockAddr(); } socklen_t sockAddrLen() const override { return instance_.sockAddrLen(); } Address::Type type() const override { return instance_.type(); } - const std::string& socketInterface() const override { return socket_interface_; } + const SocketInterface& socketInterface() const override { + return SocketInterfaceSingleton::get(); + } private: std::string antagonistic_name_; Address::Ipv4Instance instance_; - std::string socket_interface_{""}; }; TEST_F(DnsImplConstructor, SupportCustomAddressInstances) { diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index ef1686bcc688..b9d6607567d2 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -152,8 +152,8 @@ TEST_F(HttpConnManFinalizerImplTest, OriginalAndLongPath) { const std::string path_prefix = "http://"; const std::string expected_path(256, 'a'); const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; Http::TestRequestHeaderMapImpl request_headers{{"x-request-id", "id"}, {"x-envoy-original-path", path}, @@ -187,8 +187,8 @@ TEST_F(HttpConnManFinalizerImplTest, NoGeneratedId) { const std::string path_prefix = "http://"; const std::string expected_path(256, 'a'); const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; Http::TestRequestHeaderMapImpl request_headers{{":path", ""}, {"x-envoy-original-path", path}, @@ -221,8 +221,8 @@ TEST_F(HttpConnManFinalizerImplTest, Connect) { const std::string path_prefix = "http://"; const std::string expected_path(256, 'a'); const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; Http::TestRequestHeaderMapImpl request_headers{{":method", "CONNECT"}, {"x-forwarded-proto", "http"}}; @@ -352,8 +352,8 @@ TEST_F(HttpConnManFinalizerImplTest, SpanOptionalHeaders) { Http::TestResponseHeaderMapImpl response_headers; Http::TestResponseTrailerMapImpl response_trailers; const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; absl::optional protocol = Http::Protocol::Http10; EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); @@ -537,8 +537,8 @@ TEST_F(HttpConnManFinalizerImplTest, SpanPopulatedFailureResponse) { Http::TestResponseHeaderMapImpl response_headers; Http::TestResponseTrailerMapImpl response_trailers; const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; request_headers.setHost("api"); request_headers.setUserAgent("agent"); @@ -587,8 +587,8 @@ TEST_F(HttpConnManFinalizerImplTest, SpanPopulatedFailureResponse) { TEST_F(HttpConnManFinalizerImplTest, GrpcOkStatus) { const std::string path_prefix = "http://"; const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; Http::TestRequestHeaderMapImpl request_headers{{":method", "POST"}, {":scheme", "http"}, @@ -637,8 +637,8 @@ TEST_F(HttpConnManFinalizerImplTest, GrpcOkStatus) { TEST_F(HttpConnManFinalizerImplTest, GrpcErrorTag) { const std::string path_prefix = "http://"; const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; Http::TestRequestHeaderMapImpl request_headers{{":method", "POST"}, {":scheme", "http"}, @@ -683,8 +683,8 @@ TEST_F(HttpConnManFinalizerImplTest, GrpcErrorTag) { TEST_F(HttpConnManFinalizerImplTest, GrpcTrailersOnly) { const std::string path_prefix = "http://"; const std::string expected_ip = "10.0.0.100"; - const auto remote_address = - Network::Address::InstanceConstSharedPtr{new Network::Address::Ipv4Instance(expected_ip, 0)}; + const auto remote_address = Network::Address::InstanceConstSharedPtr{ + new Network::Address::Ipv4Instance(expected_ip, 0, nullptr)}; Http::TestRequestHeaderMapImpl request_headers{{":method", "POST"}, {":scheme", "http"}, diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 170246b40eb9..6f985930d05b 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -625,8 +625,8 @@ TEST_F(HttpConnectionManagerConfigTest, UnixSocketInternalAddress) { scoped_routes_config_provider_manager_, http_tracer_manager_, filter_config_provider_manager_); Network::Address::PipeInstance unixAddress{"/foo"}; - Network::Address::Ipv4Instance internalIpAddress{"127.0.0.1", 0}; - Network::Address::Ipv4Instance externalIpAddress{"12.0.0.1", 0}; + Network::Address::Ipv4Instance internalIpAddress{"127.0.0.1", 0, nullptr}; + Network::Address::Ipv4Instance externalIpAddress{"12.0.0.1", 0, nullptr}; EXPECT_TRUE(config.internalAddressConfig().isInternalAddress(unixAddress)); EXPECT_TRUE(config.internalAddressConfig().isInternalAddress(internalIpAddress)); EXPECT_FALSE(config.internalAddressConfig().isInternalAddress(externalIpAddress)); diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 76f3a16b56b1..2eb348a432ec 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -4559,11 +4559,11 @@ TEST_P(SslReadBufferLimitTest, TestBind) { std::string address_string = TestUtility::getIpv4Loopback(); if (GetParam() == Network::Address::IpVersion::v4) { source_address_ = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv4Instance(address_string, 0)}; + new Network::Address::Ipv4Instance(address_string, 0, nullptr)}; } else { address_string = "::1"; source_address_ = Network::Address::InstanceConstSharedPtr{ - new Network::Address::Ipv6Instance(address_string, 0)}; + new Network::Address::Ipv6Instance(address_string, 0, nullptr)}; } initialize(); diff --git a/test/integration/socket_interface_integration_test.cc b/test/integration/socket_interface_integration_test.cc index 3e40a901d6c3..8969d41a55a1 100644 --- a/test/integration/socket_interface_integration_test.cc +++ b/test/integration/socket_interface_integration_test.cc @@ -68,11 +68,12 @@ TEST_P(SocketInterfaceIntegrationTest, AddressWithSocketInterface) { ConnectionStatusCallbacks connect_callbacks_; Network::ClientConnectionPtr client_; + const Network::SocketInterface* sock_interface = Network::socketInterface( + "envoy.extensions.network.socket_interface.default_socket_interface"); Network::Address::InstanceConstSharedPtr address = std::make_shared( Network::Test::getLoopbackAddressUrlString(Network::Address::IpVersion::v4), - lookupPort("listener_0"), - "envoy.extensions.network.socket_interface.default_socket_interface"); + lookupPort("listener_0"), sock_interface); client_ = dispatcher_->createClientConnection(address, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 45371be5d584..144051c0d981 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -15,6 +15,7 @@ #include "envoy/stats/scope.h" #include "common/network/filter_manager_impl.h" +#include "common/network/socket_interface.h" #include "common/stats/isolated_store_impl.h" #include "test/mocks/event/mocks.h" @@ -442,11 +443,12 @@ class MockResolvedAddress : public Address::Instance { const std::string& asString() const override { return physical_; } absl::string_view asStringView() const override { return physical_; } const std::string& logicalName() const override { return logical_; } - const std::string& socketInterface() const override { return socket_interface_; } + const Network::SocketInterface& socketInterface() const override { + return SocketInterfaceSingleton::get(); + } const std::string logical_; const std::string physical_; - const std::string socket_interface_{""}; }; class MockTransportSocketCallbacks : public TransportSocketCallbacks { diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index ebcc66a8c656..a53c44d6e341 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -584,7 +584,7 @@ TEST_F(ConnectionHandlerTest, FallbackToWildcardListener) { })); // Zero port to match the port of AnyAddress Network::Address::InstanceConstSharedPtr alt_address( - new Network::Address::Ipv4Instance("127.0.0.2", 0)); + new Network::Address::Ipv4Instance("127.0.0.2", 0, nullptr)); EXPECT_CALL(*test_filter, onAccept(_)) .WillOnce(Invoke([&](Network::ListenerFilterCallbacks& cb) -> Network::FilterStatus { cb.socket().restoreLocalAddress(alt_address); From e319b7c2a93783cd8bde45fa4c002ae02a4cb13f Mon Sep 17 00:00:00 2001 From: Nicolas Flacco <47160394+FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg@users.noreply.github.com> Date: Wed, 12 Aug 2020 10:57:30 -0700 Subject: [PATCH 46/67] redis: fault injection (#12551) Un-Reverts 048583b, with fix for high cpu consumption. This PR implements fault injection for Redis; specifically delay and error faults (which themselves can have delays added). I chose not to implement a separate filter after discussing with Henry; we concluded that the faults we felt were useful didn't need many levels- just a delay on top of the original fault, if any. In addition, as the Redis protocol doesn't support headers that makes it a bit different again from Envoy's http fault injection. This PR fixes the issue previously seen with redis fault injection where excessive cpu was used on connection creation. Faults record metrics on the original request- and the delay fault adds extra latency which is included in the command latency for that request. Also, faults can apply only to certain commands. Future work: Add several other faults, including cache misses and connection failures. Signed-off-by: FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg --- .../network/redis_proxy/v3/redis_proxy.proto | 56 ++++- .../network_filters/redis_proxy_filter.rst | 52 ++++- docs/root/version_history/current.rst | 1 + .../network/redis_proxy/v3/redis_proxy.proto | 56 ++++- .../filters/network/common/redis/BUILD | 22 ++ .../filters/network/common/redis/fault.h | 54 +++++ .../network/common/redis/fault_impl.cc | 148 +++++++++++++ .../filters/network/common/redis/fault_impl.h | 108 +++++++++ .../filters/network/redis_proxy/BUILD | 3 + .../network/redis_proxy/command_splitter.h | 5 +- .../redis_proxy/command_splitter_impl.cc | 123 +++++++++-- .../redis_proxy/command_splitter_impl.h | 131 ++++++++--- .../filters/network/redis_proxy/config.cc | 6 +- .../network/redis_proxy/proxy_filter.cc | 3 +- .../network/redis_proxy/proxy_filter.h | 2 + .../filters/network/common/redis/BUILD | 13 ++ .../network/common/redis/fault_test.cc | 206 ++++++++++++++++++ .../filters/network/redis_proxy/BUILD | 9 + .../redis_proxy/command_lookup_speed_test.cc | 15 +- .../redis_proxy/command_splitter_impl_test.cc | 196 +++++++++++++++-- .../network/redis_proxy/config_test.cc | 37 ++++ .../filters/network/redis_proxy/mocks.cc | 4 + .../filters/network/redis_proxy/mocks.h | 19 +- .../network/redis_proxy/proxy_filter_test.cc | 112 +++++----- .../redis_proxy_integration_test.cc | 56 ++++- 25 files changed, 1292 insertions(+), 145 deletions(-) create mode 100644 source/extensions/filters/network/common/redis/fault.h create mode 100644 source/extensions/filters/network/common/redis/fault_impl.cc create mode 100644 source/extensions/filters/network/common/redis/fault_impl.h create mode 100644 test/extensions/filters/network/common/redis/fault_test.cc diff --git a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto index af69d33a6340..402937fff28f 100644 --- a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto +++ b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto @@ -23,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Redis Proxy :ref:`configuration overview `. // [#extension: envoy.filters.network.redis_proxy] -// [#next-free-field: 8] +// [#next-free-field: 9] message RedisProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.redis_proxy.v2.RedisProxy"; @@ -183,6 +183,31 @@ message RedisProxy { Route catch_all_route = 4; } + // RedisFault defines faults used for fault injection. + message RedisFault { + enum RedisFaultType { + // Delays requests. This is the base fault; other faults can have delays added. + DELAY = 0; + + // Returns errors on requests. + ERROR = 1; + } + + // Fault type. + RedisFaultType fault_type = 1 [(validate.rules).enum = {defined_only: true}]; + + // Percentage of requests fault applies to. + config.core.v3.RuntimeFractionalPercent fault_enabled = 2 + [(validate.rules).message = {required: true}]; + + // Delay for all faults. If not set, defaults to zero + google.protobuf.Duration delay = 3; + + // Commands fault is restricted to, if any. If not set, fault applies to all commands + // other than auth and ping (due to special handling of those commands in Envoy). + repeated string commands = 4; + } + reserved 2; reserved "cluster"; @@ -236,6 +261,35 @@ message RedisProxy { // AUTH, but no password is set" error will be returned. config.core.v3.DataSource downstream_auth_password = 6 [(udpa.annotations.sensitive) = true]; + // List of faults to inject. Faults currently come in two flavors: + // - Delay, which delays a request. + // - Error, which responds to a request with an error. Errors can also have delays attached. + // + // Example: + // + // .. code-block:: yaml + // + // faults: + // - fault_type: ERROR + // fault_enabled: + // default_value: + // numerator: 10 + // denominator: HUNDRED + // runtime_key: "bogus_key" + // commands: + // - GET + // - fault_type: DELAY + // fault_enabled: + // default_value: + // numerator: 10 + // denominator: HUNDRED + // runtime_key: "bogus_key" + // delay: 2s + // + // See the :ref:`fault injection section + // ` for more information on how to configure this. + repeated RedisFault faults = 8; + // If a username is provided an ACL style AUTH command will be required with a username and password. // Authenticate Redis client connections locally by forcing downstream clients to issue a `Redis // AUTH command `_ with this username and the *downstream_auth_password* diff --git a/docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst index 3c3fb77f3861..6adf7c8ffb27 100644 --- a/docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst +++ b/docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst @@ -58,7 +58,9 @@ changed to microseconds by setting the configuration parameter :ref:`latency_in_ total, Counter, Number of commands success, Counter, Number of commands that were successful error, Counter, Number of commands that returned a partial or complete error response - latency, Histogram, Command execution time in milliseconds + latency, Histogram, Command execution time in milliseconds (including delay faults) + error_fault, Counter, Number of commands that had an error fault injected + delay_fault, Counter, Number of commands that had a delay fault injected .. _config_network_filters_redis_proxy_per_command_stats: @@ -70,3 +72,51 @@ The Redis proxy filter supports the following runtime settings: redis.drain_close_enabled % of connections that will be drain closed if the server is draining and would otherwise attempt a drain close. Defaults to 100. + +.. _config_network_filters_redis_proxy_fault_injection: + +Fault Injection +--------------- + +The Redis filter can perform fault injection. Currently, Delay and Error faults are supported. +Delay faults delay a request, and Error faults respond with an error. Moreover, errors can be delayed. + +Note that the Redis filter does not check for correctness in your configuration - it is the user's +responsibility to make sure both the default and runtime percentages are correct! This is because +percentages can be changed during runtime, and validating correctness at request time is expensive. +If multiple faults are specified, the fault injection percentage should not exceed 100% for a given +fault and Redis command combination. For example, if two faults are specified; one applying to GET at 60 +%, and one applying to all commands at 50%, that is a bad configuration as GET now has 110% chance of +applying a fault. This means that every request will have a fault. + +If a delay is injected, the delay is additive - if the request took 400ms and a delay of 100ms +is injected, then the total request latency is 500ms. Also, due to implementation of the redis protocol, +a delayed request will delay everything that comes in after it, due to the proxy's need to respect the +order of commands it receives. + +Note that faults must have a `fault_enabled` field, and are not enabled by default (if no default value +or runtime key are set). + +Example configuration: + +.. code-block:: yaml + + faults: + - fault_type: ERROR + fault_enabled: + default_value: + numerator: 10 + denominator: HUNDRED + runtime_key: "bogus_key" + commands: + - GET + - fault_type: DELAY + fault_enabled: + default_value: + numerator: 10 + denominator: HUNDRED + runtime_key: "bogus_key" + delay: 2s + +This creates two faults- an error, applying only to GET commands at 10%, and a delay, applying to all +commands at 10%. This means that 20% of GET commands will have a fault applied, as discussed earlier. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 81d351831c25..9ca7c87a0c55 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -62,6 +62,7 @@ New Features * postgres network filter: :ref:`metadata ` is produced based on SQL query. * ratelimit: added :ref:`enable_x_ratelimit_headers ` option to enable `X-RateLimit-*` headers as defined in `draft RFC `_. * rbac filter: added a log action to the :ref:`RBAC filter ` which sets dynamic metadata to inform access loggers whether to log. +* redis: added fault injection support :ref:`fault injection for redis proxy `, described further in :ref:`configuration documentation `. * router: added new :ref:`envoy-ratelimited` retry policy, which allows retrying envoy's own rate limited responses. diff --git a/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto b/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto index 8f996c30f9ae..0bc52493bb29 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto @@ -23,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // Redis Proxy :ref:`configuration overview `. // [#extension: envoy.filters.network.redis_proxy] -// [#next-free-field: 8] +// [#next-free-field: 9] message RedisProxy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.network.redis_proxy.v2.RedisProxy"; @@ -182,6 +182,31 @@ message RedisProxy { [deprecated = true, (envoy.annotations.disallowed_by_default) = true]; } + // RedisFault defines faults used for fault injection. + message RedisFault { + enum RedisFaultType { + // Delays requests. This is the base fault; other faults can have delays added. + DELAY = 0; + + // Returns errors on requests. + ERROR = 1; + } + + // Fault type. + RedisFaultType fault_type = 1 [(validate.rules).enum = {defined_only: true}]; + + // Percentage of requests fault applies to. + config.core.v3.RuntimeFractionalPercent fault_enabled = 2 + [(validate.rules).message = {required: true}]; + + // Delay for all faults. If not set, defaults to zero + google.protobuf.Duration delay = 3; + + // Commands fault is restricted to, if any. If not set, fault applies to all commands + // other than auth and ping (due to special handling of those commands in Envoy). + repeated string commands = 4; + } + // The prefix to use when emitting :ref:`statistics `. string stat_prefix = 1 [(validate.rules).string = {min_bytes: 1}]; @@ -231,6 +256,35 @@ message RedisProxy { // AUTH, but no password is set" error will be returned. config.core.v3.DataSource downstream_auth_password = 6 [(udpa.annotations.sensitive) = true]; + // List of faults to inject. Faults currently come in two flavors: + // - Delay, which delays a request. + // - Error, which responds to a request with an error. Errors can also have delays attached. + // + // Example: + // + // .. code-block:: yaml + // + // faults: + // - fault_type: ERROR + // fault_enabled: + // default_value: + // numerator: 10 + // denominator: HUNDRED + // runtime_key: "bogus_key" + // commands: + // - GET + // - fault_type: DELAY + // fault_enabled: + // default_value: + // numerator: 10 + // denominator: HUNDRED + // runtime_key: "bogus_key" + // delay: 2s + // + // See the :ref:`fault injection section + // ` for more information on how to configure this. + repeated RedisFault faults = 8; + // If a username is provided an ACL style AUTH command will be required with a username and password. // Authenticate Redis client connections locally by forcing downstream clients to issue a `Redis // AUTH command `_ with this username and the *downstream_auth_password* diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 5c0393d36a62..497286700ff2 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -100,3 +100,25 @@ envoy_cc_library( "//source/common/stats:utility_lib", ], ) + +envoy_cc_library( + name = "fault_interface", + hdrs = ["fault.h"], + deps = [ + "@envoy_api//envoy/type/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "fault_lib", + srcs = ["fault_impl.cc"], + hdrs = ["fault_impl.h"], + deps = [ + ":codec_lib", + ":fault_interface", + "//include/envoy/common:random_generator_interface", + "//include/envoy/upstream:upstream_interface", + "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/filters/network/common/redis/fault.h b/source/extensions/filters/network/common/redis/fault.h new file mode 100644 index 000000000000..8ab4886f9357 --- /dev/null +++ b/source/extensions/filters/network/common/redis/fault.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +#include "envoy/common/pure.h" +#include "envoy/type/v3/percent.pb.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + +/** + * Fault Type. + */ +enum class FaultType { Delay, Error }; + +class Fault { +public: + virtual ~Fault() = default; + + virtual FaultType faultType() const PURE; + virtual std::chrono::milliseconds delayMs() const PURE; + virtual const std::vector commands() const PURE; + virtual envoy::type::v3::FractionalPercent defaultValue() const PURE; + virtual absl::optional runtimeKey() const PURE; +}; + +using FaultSharedPtr = std::shared_ptr; + +class FaultManager { +public: + virtual ~FaultManager() = default; + + /** + * Get fault type and delay given a Redis command. + * @param command supplies the Redis command string. + */ + virtual const Fault* getFaultForCommand(const std::string& command) const PURE; +}; + +using FaultManagerPtr = std::unique_ptr; + +using FaultManagerSharedPtr = std::shared_ptr; + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/common/redis/fault_impl.cc b/source/extensions/filters/network/common/redis/fault_impl.cc new file mode 100644 index 000000000000..ce1174968513 --- /dev/null +++ b/source/extensions/filters/network/common/redis/fault_impl.cc @@ -0,0 +1,148 @@ +#include "extensions/filters/network/common/redis/fault_impl.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + +struct FaultManagerKeyNamesValues { + // The rbac filter rejected the request + const std::string AllKey = "ALL_KEY"; +}; +using FaultManagerKeyNames = ConstSingleton; + +FaultManagerImpl::FaultImpl::FaultImpl( + envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault base_fault) + : commands_(buildCommands(base_fault)) { + delay_ms_ = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(base_fault, delay, 0)); + + switch (base_fault.fault_type()) { + case envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::RedisFault::DELAY: + fault_type_ = FaultType::Delay; + break; + case envoy::extensions::filters::network::redis_proxy::v3::RedisProxy::RedisFault::ERROR: + fault_type_ = FaultType::Error; + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + break; + } + + default_value_ = base_fault.fault_enabled().default_value(); + runtime_key_ = base_fault.fault_enabled().runtime_key(); +}; + +std::vector FaultManagerImpl::FaultImpl::buildCommands( + envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault base_fault) { + std::vector commands; + for (const std::string& command : base_fault.commands()) { + commands.emplace_back(absl::AsciiStrToLower(command)); + } + return commands; +} + +FaultManagerImpl::FaultManagerImpl( + Random::RandomGenerator& random, Runtime::Loader& runtime, + const Protobuf::RepeatedPtrField< + ::envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault> + faults) + : fault_map_(buildFaultMap(faults)), random_(random), runtime_(runtime) {} + +FaultMap FaultManagerImpl::buildFaultMap( + const Protobuf::RepeatedPtrField< + ::envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault> + faults) { + // Next, create the fault map that maps commands to pointers to Fault objects. + // Group faults by command + FaultMap fault_map; + for (auto const& base_fault : faults) { + auto fault_ptr = std::make_shared(base_fault); + if (!fault_ptr->commands().empty()) { + for (const std::string& command : fault_ptr->commands()) { + fault_map[command].emplace_back(fault_ptr); + } + } else { + // Generic "ALL" entry in map for faults that map to all keys; also add to each command + fault_map[FaultManagerKeyNames::get().AllKey].emplace_back(fault_ptr); + } + } + + // Add the ALL keys faults to each command too so that we can just query faults by command. + // Get all ALL_KEY faults. + FaultMap::iterator it_outer = fault_map.find(FaultManagerKeyNames::get().AllKey); + if (it_outer != fault_map.end()) { + for (const FaultSharedPtr& fault_ptr : it_outer->second) { + FaultMap::iterator it_inner; + for (it_inner = fault_map.begin(); it_inner != fault_map.end(); it_inner++) { + std::string command = it_inner->first; + if (command != FaultManagerKeyNames::get().AllKey) { + fault_map[command].push_back(fault_ptr); + } + } + } + } + return fault_map; +} + +uint64_t FaultManagerImpl::getIntegerNumeratorOfFractionalPercent( + absl::string_view key, const envoy::type::v3::FractionalPercent& default_value) const { + uint64_t numerator; + if (default_value.denominator() == envoy::type::v3::FractionalPercent::HUNDRED) { + numerator = default_value.numerator(); + } else { + int denominator = + ProtobufPercentHelper::fractionalPercentDenominatorToInt(default_value.denominator()); + numerator = (default_value.numerator() * 100) / denominator; + } + return runtime_.snapshot().getInteger(key, numerator); +} + +// Fault checking algorithm: +// +// For example, if we have an ERROR fault at 5% for all commands, and a DELAY fault at 10% for GET, +// if we receive a GET, we want 5% of GETs to get DELAY, and 10% to get ERROR. Thus, we need to +// amortize the percentages. +// +// 0. Get random number. +// 1. Get faults for given command. +// 2. For each fault, calculate the amortized fault injection percentage. +// +// Note that we do not check to make sure the probabilities of faults are <= 100%! +const Fault* FaultManagerImpl::getFaultForCommandInternal(const std::string& command) const { + FaultMap::const_iterator it_outer = fault_map_.find(command); + if (it_outer != fault_map_.end()) { + auto random_number = random_.random() % 100; + int amortized_fault = 0; + + for (const FaultSharedPtr& fault_ptr : it_outer->second) { + uint64_t fault_injection_percentage = getIntegerNumeratorOfFractionalPercent( + fault_ptr->runtimeKey().value(), fault_ptr->defaultValue()); + if (random_number < (fault_injection_percentage + amortized_fault)) { + return fault_ptr.get(); + } else { + amortized_fault += fault_injection_percentage; + } + } + } + + return nullptr; +} + +const Fault* FaultManagerImpl::getFaultForCommand(const std::string& command) const { + if (!fault_map_.empty()) { + if (fault_map_.count(command) > 0) { + return getFaultForCommandInternal(command); + } else { + return getFaultForCommandInternal(FaultManagerKeyNames::get().AllKey); + } + } + + return nullptr; +} + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/common/redis/fault_impl.h b/source/extensions/filters/network/common/redis/fault_impl.h new file mode 100644 index 000000000000..3850a8a4b4c9 --- /dev/null +++ b/source/extensions/filters/network/common/redis/fault_impl.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include + +#include "envoy/api/api.h" +#include "envoy/common/random_generator.h" +#include "envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.pb.h" +#include "envoy/upstream/upstream.h" + +#include "common/protobuf/utility.h" +#include "common/singleton/const_singleton.h" + +#include "extensions/filters/network/common/redis/fault.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + +using FaultMap = absl::flat_hash_map>; + +/** + * Message returned for particular types of faults. + */ +struct FaultMessagesValues { + const std::string Error = "Fault Injected: Error"; +}; +using FaultMessages = ConstSingleton; + +/** + * Fault management- creation, storage and retrieval. Faults are queried for by command, + * so they are stored in an unordered map using the command as key. For faults that apply to + * all commands, we use a special ALL_KEYS entry in the map. + */ +class FaultManagerImpl : public FaultManager { +public: + FaultManagerImpl( + Random::RandomGenerator& random, Runtime::Loader& runtime, + const Protobuf::RepeatedPtrField< + ::envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault> + base_faults); + + const Fault* getFaultForCommand(const std::string& command) const override; + + static FaultSharedPtr makeFaultForTest(Common::Redis::FaultType fault_type, + std::chrono::milliseconds delay_ms) { + envoy::type::v3::FractionalPercent default_value; + default_value.set_numerator(100); + default_value.set_denominator(envoy::type::v3::FractionalPercent::HUNDRED); + FaultImpl fault = + FaultImpl(fault_type, delay_ms, std::vector(), default_value, "foo"); + return std::make_shared(fault); + } + + // Allow the unit test to have access to private members. + friend class FaultTest; + +private: + class FaultImpl : public Fault { + public: + FaultImpl( + envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault base_fault); + FaultImpl(FaultType fault_type, std::chrono::milliseconds delay_ms, + const std::vector commands, + envoy::type::v3::FractionalPercent default_value, + absl::optional runtime_key) + : fault_type_(fault_type), delay_ms_(delay_ms), commands_(commands), + default_value_(default_value), runtime_key_(runtime_key) {} // For testing only + + FaultType faultType() const override { return fault_type_; }; + std::chrono::milliseconds delayMs() const override { return delay_ms_; }; + const std::vector commands() const override { return commands_; }; + envoy::type::v3::FractionalPercent defaultValue() const override { return default_value_; }; + absl::optional runtimeKey() const override { return runtime_key_; }; + + private: + static std::vector buildCommands( + envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault base_fault); + + FaultType fault_type_; + std::chrono::milliseconds delay_ms_; + const std::vector commands_; + envoy::type::v3::FractionalPercent default_value_; + absl::optional runtime_key_; + }; + + static FaultMap + buildFaultMap(const Protobuf::RepeatedPtrField< + ::envoy::extensions::filters::network::redis_proxy::v3::RedisProxy_RedisFault> + faults); + + uint64_t getIntegerNumeratorOfFractionalPercent( + absl::string_view key, const envoy::type::v3::FractionalPercent& default_value) const; + const Fault* getFaultForCommandInternal(const std::string& command) const; + const FaultMap fault_map_; + +protected: + Random::RandomGenerator& random_; + Runtime::Loader& runtime_; +}; + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/redis_proxy/BUILD b/source/extensions/filters/network/redis_proxy/BUILD index c0b742efa02e..a2163f563e27 100644 --- a/source/extensions/filters/network/redis_proxy/BUILD +++ b/source/extensions/filters/network/redis_proxy/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( name = "command_splitter_interface", hdrs = ["command_splitter.h"], deps = [ + "//include/envoy/event:dispatcher_interface", "//source/extensions/filters/network/common/redis:codec_interface", ], ) @@ -64,6 +65,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/stats:timespan_lib", "//source/extensions/filters/network/common/redis:client_lib", + "//source/extensions/filters/network/common/redis:fault_lib", "//source/extensions/filters/network/common/redis:supported_commands_lib", "//source/extensions/filters/network/common/redis:utility_lib", ], @@ -130,6 +132,7 @@ envoy_cc_extension( "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", "//source/extensions/filters/network/common/redis:codec_lib", + "//source/extensions/filters/network/common/redis:fault_lib", "//source/extensions/filters/network/common/redis:redis_command_stats_lib", "//source/extensions/filters/network/redis_proxy:command_splitter_lib", "//source/extensions/filters/network/redis_proxy:conn_pool_lib", diff --git a/source/extensions/filters/network/redis_proxy/command_splitter.h b/source/extensions/filters/network/redis_proxy/command_splitter.h index e03d0a92e137..019ae429459a 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter.h +++ b/source/extensions/filters/network/redis_proxy/command_splitter.h @@ -3,6 +3,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/event/dispatcher.h" #include "extensions/filters/network/common/redis/codec.h" @@ -72,12 +73,14 @@ class Instance { * Make a split redis request capable of being retried/redirected. * @param request supplies the split request to make (ownership transferred to call). * @param callbacks supplies the split request completion callbacks. + * @param dispatcher supplies dispatcher used for delay fault timer. * @return SplitRequestPtr a handle to the active request or nullptr if the request has already * been satisfied (via onResponse() being called). The splitter ALWAYS calls * onResponse() for a given request. */ virtual SplitRequestPtr makeRequest(Common::Redis::RespValuePtr&& request, - SplitCallbacks& callbacks) PURE; + SplitCallbacks& callbacks, + Event::Dispatcher& dispatcher) PURE; }; } // namespace CommandSplitter diff --git a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc index a5bd89588f51..18b697d1c700 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc +++ b/source/extensions/filters/network/redis_proxy/command_splitter_impl.cc @@ -79,7 +79,9 @@ void SplitRequestBase::updateStats(const bool success) { } else { command_stats_.error_.inc(); } - command_latency_->complete(); + if (command_latency_ != nullptr) { + command_latency_->complete(); + } } SingleServerRequest::~SingleServerRequest() { ASSERT(!handle_); } @@ -90,10 +92,12 @@ void SingleServerRequest::onResponse(Common::Redis::RespValuePtr&& response) { callbacks_.onResponse(std::move(response)); } -void SingleServerRequest::onFailure() { +void SingleServerRequest::onFailure() { onFailure(Response::get().UpstreamFailure); } + +void SingleServerRequest::onFailure(std::string error_msg) { handle_ = nullptr; updateStats(false); - callbacks_.onResponse(Common::Redis::Utility::makeError(Response::get().UpstreamFailure)); + callbacks_.onResponse(Common::Redis::Utility::makeError(error_msg)); } void SingleServerRequest::cancel() { @@ -101,13 +105,44 @@ void SingleServerRequest::cancel() { handle_ = nullptr; } +SplitRequestPtr ErrorFaultRequest::create(SplitCallbacks& callbacks, CommandStats& command_stats, + TimeSource& time_source, bool delay_command_latency) { + std::unique_ptr request_ptr{ + new ErrorFaultRequest(callbacks, command_stats, time_source, delay_command_latency)}; + + request_ptr->onFailure(Common::Redis::FaultMessages::get().Error); + command_stats.error_fault_.inc(); + return nullptr; +} + +std::unique_ptr DelayFaultRequest::create(SplitCallbacks& callbacks, + CommandStats& command_stats, + TimeSource& time_source, + Event::Dispatcher& dispatcher, + std::chrono::milliseconds delay) { + return std::make_unique(callbacks, command_stats, time_source, dispatcher, + delay); +} + +void DelayFaultRequest::onResponse(Common::Redis::RespValuePtr&& response) { + response_ = std::move(response); + delay_timer_->enableTimer(delay_); +} + +void DelayFaultRequest::onDelayResponse() { + command_stats_.delay_fault_.inc(); + command_latency_->complete(); // Complete latency of the command stats of the wrapped request + callbacks_.onResponse(std::move(response_)); +} + +void DelayFaultRequest::cancel() { delay_timer_->disableTimer(); } + SplitRequestPtr SimpleRequest::create(Router& router, Common::Redis::RespValuePtr&& incoming_request, SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source) { + TimeSource& time_source, bool delay_command_latency) { std::unique_ptr request_ptr{ - new SimpleRequest(callbacks, command_stats, time_source)}; - + new SimpleRequest(callbacks, command_stats, time_source, delay_command_latency)}; const auto route = router.upstreamPool(incoming_request->asArray()[1].asString()); if (route) { Common::Redis::RespValueSharedPtr base_request = std::move(incoming_request); @@ -126,7 +161,7 @@ SplitRequestPtr SimpleRequest::create(Router& router, SplitRequestPtr EvalRequest::create(Router& router, Common::Redis::RespValuePtr&& incoming_request, SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source) { + TimeSource& time_source, bool delay_command_latency) { // EVAL looks like: EVAL script numkeys key [key ...] arg [arg ...] // Ensure there are at least three args to the command or it cannot be hashed. if (incoming_request->asArray().size() < 4) { @@ -135,7 +170,8 @@ SplitRequestPtr EvalRequest::create(Router& router, Common::Redis::RespValuePtr& return nullptr; } - std::unique_ptr request_ptr{new EvalRequest(callbacks, command_stats, time_source)}; + std::unique_ptr request_ptr{ + new EvalRequest(callbacks, command_stats, time_source, delay_command_latency)}; const auto route = router.upstreamPool(incoming_request->asArray()[3].asString()); if (route) { @@ -177,8 +213,9 @@ void FragmentedRequest::onChildFailure(uint32_t index) { SplitRequestPtr MGETRequest::create(Router& router, Common::Redis::RespValuePtr&& incoming_request, SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source) { - std::unique_ptr request_ptr{new MGETRequest(callbacks, command_stats, time_source)}; + TimeSource& time_source, bool delay_command_latency) { + std::unique_ptr request_ptr{ + new MGETRequest(callbacks, command_stats, time_source, delay_command_latency)}; request_ptr->num_pending_responses_ = incoming_request->asArray().size() - 1; request_ptr->pending_requests_.reserve(request_ptr->num_pending_responses_); @@ -250,13 +287,14 @@ void MGETRequest::onChildResponse(Common::Redis::RespValuePtr&& value, uint32_t SplitRequestPtr MSETRequest::create(Router& router, Common::Redis::RespValuePtr&& incoming_request, SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source) { + TimeSource& time_source, bool delay_command_latency) { if ((incoming_request->asArray().size() - 1) % 2 != 0) { onWrongNumberOfArguments(callbacks, *incoming_request); command_stats.error_.inc(); return nullptr; } - std::unique_ptr request_ptr{new MSETRequest(callbacks, command_stats, time_source)}; + std::unique_ptr request_ptr{ + new MSETRequest(callbacks, command_stats, time_source, delay_command_latency)}; request_ptr->num_pending_responses_ = (incoming_request->asArray().size() - 1) / 2; request_ptr->pending_requests_.reserve(request_ptr->num_pending_responses_); @@ -321,13 +359,12 @@ void MSETRequest::onChildResponse(Common::Redis::RespValuePtr&& value, uint32_t } } -SplitRequestPtr SplitKeysSumResultRequest::create(Router& router, - Common::Redis::RespValuePtr&& incoming_request, - SplitCallbacks& callbacks, - CommandStats& command_stats, - TimeSource& time_source) { +SplitRequestPtr +SplitKeysSumResultRequest::create(Router& router, Common::Redis::RespValuePtr&& incoming_request, + SplitCallbacks& callbacks, CommandStats& command_stats, + TimeSource& time_source, bool delay_command_latency) { std::unique_ptr request_ptr{ - new SplitKeysSumResultRequest(callbacks, command_stats, time_source)}; + new SplitKeysSumResultRequest(callbacks, command_stats, time_source, delay_command_latency)}; request_ptr->num_pending_responses_ = incoming_request->asArray().size() - 1; request_ptr->pending_requests_.reserve(request_ptr->num_pending_responses_); @@ -392,12 +429,13 @@ void SplitKeysSumResultRequest::onChildResponse(Common::Redis::RespValuePtr&& va } InstanceImpl::InstanceImpl(RouterPtr&& router, Stats::Scope& scope, const std::string& stat_prefix, - TimeSource& time_source, bool latency_in_micros) + TimeSource& time_source, bool latency_in_micros, + Common::Redis::FaultManagerPtr&& fault_manager) : router_(std::move(router)), simple_command_handler_(*router_), eval_command_handler_(*router_), mget_handler_(*router_), mset_handler_(*router_), split_keys_sum_result_handler_(*router_), stats_{ALL_COMMAND_SPLITTER_STATS(POOL_COUNTER_PREFIX(scope, stat_prefix + "splitter."))}, - time_source_(time_source) { + time_source_(time_source), fault_manager_(std::move(fault_manager)) { for (const std::string& command : Common::Redis::SupportedCommands::simpleCommands()) { addHandler(scope, stat_prefix, command, latency_in_micros, simple_command_handler_); } @@ -419,7 +457,8 @@ InstanceImpl::InstanceImpl(RouterPtr&& router, Stats::Scope& scope, const std::s } SplitRequestPtr InstanceImpl::makeRequest(Common::Redis::RespValuePtr&& request, - SplitCallbacks& callbacks) { + SplitCallbacks& callbacks, + Event::Dispatcher& dispatcher) { if ((request->type() != Common::Redis::RespType::Array) || request->asArray().empty()) { onInvalidRequest(callbacks); return nullptr; @@ -468,6 +507,7 @@ SplitRequestPtr InstanceImpl::makeRequest(Common::Redis::RespValuePtr&& request, return nullptr; } + // Get the handler for the downstream request auto handler = handler_lookup_table_.find(to_lower_string.c_str()); if (handler == nullptr) { stats_.unsupported_command_.inc(); @@ -475,11 +515,46 @@ SplitRequestPtr InstanceImpl::makeRequest(Common::Redis::RespValuePtr&& request, fmt::format("unsupported command '{}'", request->asArray()[0].asString()))); return nullptr; } + + // Fault Injection Check + const Common::Redis::Fault* fault_ptr = fault_manager_->getFaultForCommand(to_lower_string); + + // Check if delay, which determines which callbacks to use. If a delay fault is enabled, + // the delay fault itself wraps the request (or other fault) and the delay fault itself + // implements the callbacks functions, and in turn calls the real callbacks after injecting + // delay on the result of the wrapped request or fault. + const bool has_delay_fault = + fault_ptr != nullptr && fault_ptr->delayMs() > std::chrono::milliseconds(0); + std::unique_ptr delay_fault_ptr; + if (has_delay_fault) { + delay_fault_ptr = DelayFaultRequest::create(callbacks, handler->command_stats_, time_source_, + dispatcher, fault_ptr->delayMs()); + } + + // Note that the command_stats_ object of the original request is used for faults, so that our + // downstream metrics reflect any faults added (with special fault metrics) or extra latency from + // a delay. 2) we use a ternary operator for the callback parameter- we want to use the + // delay_fault as callback if there is a delay per the earlier comment. ENVOY_LOG(debug, "redis: splitting '{}'", request->toString()); handler->command_stats_.total_.inc(); - SplitRequestPtr request_ptr = handler->handler_.get().startRequest( - std::move(request), callbacks, handler->command_stats_, time_source_); - return request_ptr; + + SplitRequestPtr request_ptr; + if (fault_ptr != nullptr && fault_ptr->faultType() == Common::Redis::FaultType::Error) { + request_ptr = ErrorFaultRequest::create(has_delay_fault ? *delay_fault_ptr : callbacks, + handler->command_stats_, time_source_, has_delay_fault); + } else { + request_ptr = handler->handler_.get().startRequest( + std::move(request), has_delay_fault ? *delay_fault_ptr : callbacks, handler->command_stats_, + time_source_, has_delay_fault); + } + + // Complete delay, if any. The delay fault takes ownership of the wrapped request. + if (has_delay_fault) { + delay_fault_ptr->wrapped_request_ptr_ = std::move(request_ptr); + return delay_fault_ptr; + } else { + return request_ptr; + } } void InstanceImpl::onInvalidRequest(SplitCallbacks& callbacks) { diff --git a/source/extensions/filters/network/redis_proxy/command_splitter_impl.h b/source/extensions/filters/network/redis_proxy/command_splitter_impl.h index b67b4498f0cf..0263ff6b8724 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter_impl.h +++ b/source/extensions/filters/network/redis_proxy/command_splitter_impl.h @@ -10,10 +10,10 @@ #include "common/common/logger.h" #include "common/common/utility.h" -#include "common/singleton/const_singleton.h" #include "common/stats/timespan_impl.h" #include "extensions/filters/network/common/redis/client_impl.h" +#include "extensions/filters/network/common/redis/fault_impl.h" #include "extensions/filters/network/common/redis/utility.h" #include "extensions/filters/network/redis_proxy/command_splitter.h" #include "extensions/filters/network/redis_proxy/conn_pool_impl.h" @@ -42,7 +42,9 @@ using Response = ConstSingleton; #define ALL_COMMAND_STATS(COUNTER) \ COUNTER(total) \ COUNTER(success) \ - COUNTER(error) + COUNTER(error) \ + COUNTER(error_fault) \ + COUNTER(delay_fault) /** * Struct definition for all command stats. @see stats_macros.h @@ -58,7 +60,7 @@ class CommandHandler { virtual SplitRequestPtr startRequest(Common::Redis::RespValuePtr&& request, SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source) PURE; + TimeSource& time_source, bool delay_command_latency) PURE; }; class CommandHandlerBase { @@ -74,10 +76,14 @@ class SplitRequestBase : public SplitRequest { const Common::Redis::RespValue& request); void updateStats(const bool success); - SplitRequestBase(CommandStats& command_stats, TimeSource& time_source) + SplitRequestBase(CommandStats& command_stats, TimeSource& time_source, bool delay_command_latency) : command_stats_(command_stats) { - command_latency_ = std::make_unique( - command_stats_.latency_, time_source); + if (!delay_command_latency) { + command_latency_ = std::make_unique( + command_stats_.latency_, time_source); + } else { + command_latency_ = nullptr; + } } CommandStats& command_stats_; Stats::TimespanPtr command_latency_; @@ -93,14 +99,16 @@ class SingleServerRequest : public SplitRequestBase, public ConnPool::PoolCallba // ConnPool::PoolCallbacks void onResponse(Common::Redis::RespValuePtr&& response) override; void onFailure() override; + void onFailure(std::string error_msg); // RedisProxy::CommandSplitter::SplitRequest void cancel() override; protected: SingleServerRequest(SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source) - : SplitRequestBase(command_stats, time_source), callbacks_(callbacks) {} + TimeSource& time_source, bool delay_command_latency) + : SplitRequestBase(command_stats, time_source, delay_command_latency), callbacks_(callbacks) { + } SplitCallbacks& callbacks_; ConnPool::InstanceSharedPtr conn_pool_; @@ -108,6 +116,57 @@ class SingleServerRequest : public SplitRequestBase, public ConnPool::PoolCallba Common::Redis::RespValuePtr incoming_request_; }; +/** + * ErrorFaultRequest returns an error. + */ +class ErrorFaultRequest : public SingleServerRequest { +public: + static SplitRequestPtr create(SplitCallbacks& callbacks, CommandStats& command_stats, + TimeSource& time_source, bool has_delaydelay_command_latency_fault); + +private: + ErrorFaultRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source, + bool delay_command_latency) + : SingleServerRequest(callbacks, command_stats, time_source, delay_command_latency) {} +}; + +/** + * DelayFaultRequest wraps a request- either a normal request or a fault- and delays it. + */ +class DelayFaultRequest : public SplitRequestBase, public SplitCallbacks { +public: + static std::unique_ptr + create(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source, + Event::Dispatcher& dispatcher, std::chrono::milliseconds delay); + + DelayFaultRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source, + Event::Dispatcher& dispatcher, std::chrono::milliseconds delay) + : SplitRequestBase(command_stats, time_source, false), callbacks_(callbacks), delay_(delay) { + delay_timer_ = dispatcher.createTimer([this]() -> void { onDelayResponse(); }); + } + + // SplitCallbacks + bool connectionAllowed() override { return callbacks_.connectionAllowed(); } + void onAuth(const std::string& password) override { callbacks_.onAuth(password); } + void onAuth(const std::string& username, const std::string& password) override { + callbacks_.onAuth(username, password); + } + void onResponse(Common::Redis::RespValuePtr&& response) override; + + // RedisProxy::CommandSplitter::SplitRequest + void cancel() override; + + SplitRequestPtr wrapped_request_ptr_; + +private: + void onDelayResponse(); + + SplitCallbacks& callbacks_; + std::chrono::milliseconds delay_; + Event::TimerPtr delay_timer_; + Common::Redis::RespValuePtr response_; +}; + /** * SimpleRequest hashes the first argument as the key. */ @@ -115,11 +174,12 @@ class SimpleRequest : public SingleServerRequest { public: static SplitRequestPtr create(Router& router, Common::Redis::RespValuePtr&& incoming_request, SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source); + TimeSource& time_source, bool delay_command_latency); private: - SimpleRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source) - : SingleServerRequest(callbacks, command_stats, time_source) {} + SimpleRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source, + bool delay_command_latency) + : SingleServerRequest(callbacks, command_stats, time_source, delay_command_latency) {} }; /** @@ -129,11 +189,12 @@ class EvalRequest : public SingleServerRequest { public: static SplitRequestPtr create(Router& router, Common::Redis::RespValuePtr&& incoming_request, SplitCallbacks& callbacks, CommandStats& command_stats, - TimeSource& time_source); + TimeSource& time_source, bool delay_command_latency); private: - EvalRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source) - : SingleServerRequest(callbacks, command_stats, time_source) {} + EvalRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source, + bool delay_command_latency) + : SingleServerRequest(callbacks, command_stats, time_source, delay_command_latency) {} }; /** @@ -149,8 +210,10 @@ class FragmentedRequest : public SplitRequestBase { void cancel() override; protected: - FragmentedRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source) - : SplitRequestBase(command_stats, time_source), callbacks_(callbacks) {} + FragmentedRequest(SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source, + bool delay_command_latency) + : SplitRequestBase(command_stats, time_source, delay_command_latency), callbacks_(callbacks) { + } struct PendingRequest : public ConnPool::PoolCallbacks { PendingRequest(FragmentedRequest& parent, uint32_t index) : parent_(parent), index_(index) {} @@ -185,11 +248,12 @@ class MGETRequest : public FragmentedRequest, Logger::Loggable { public: InstanceImpl(RouterPtr&& router, Stats::Scope& scope, const std::string& stat_prefix, - TimeSource& time_source, bool latency_in_micros); + TimeSource& time_source, bool latency_in_micros, + Common::Redis::FaultManagerPtr&& fault_manager); // RedisProxy::CommandSplitter::Instance - SplitRequestPtr makeRequest(Common::Redis::RespValuePtr&& request, - SplitCallbacks& callbacks) override; + SplitRequestPtr makeRequest(Common::Redis::RespValuePtr&& request, SplitCallbacks& callbacks, + Event::Dispatcher& dispatcher) override; private: + friend class RedisCommandSplitterImplTest; + struct HandlerData { CommandStats command_stats_; std::reference_wrapper handler_; @@ -295,6 +365,7 @@ class InstanceImpl : public Instance, Logger::Loggable { TrieLookupTable handler_lookup_table_; InstanceStats stats_; TimeSource& time_source_; + Common::Redis::FaultManagerPtr fault_manager_; }; } // namespace CommandSplitter diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 2d62f511b393..208a1a5c1cab 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -5,6 +5,7 @@ #include "extensions/common/redis/cluster_refresh_manager_impl.h" #include "extensions/filters/network/common/redis/client_impl.h" +#include "extensions/filters/network/common/redis/fault_impl.h" #include "extensions/filters/network/redis_proxy/command_splitter_impl.h" #include "extensions/filters/network/redis_proxy/proxy_filter.h" #include "extensions/filters/network/redis_proxy/router_impl.h" @@ -86,10 +87,13 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP auto router = std::make_unique(prefix_routes, std::move(upstreams), context.runtime()); + auto fault_manager = std::make_unique( + context.random(), context.runtime(), proto_config.faults()); + std::shared_ptr splitter = std::make_shared( std::move(router), context.scope(), filter_config->stat_prefix_, context.timeSource(), - proto_config.latency_in_micros()); + proto_config.latency_in_micros(), std::move(fault_manager)); return [splitter, filter_config](Network::FilterManager& filter_manager) -> void { Common::Redis::DecoderFactoryImpl factory; filter_manager.addReadFilter(std::make_shared( diff --git a/source/extensions/filters/network/redis_proxy/proxy_filter.cc b/source/extensions/filters/network/redis_proxy/proxy_filter.cc index aa2f558cc51a..54e029813ef0 100644 --- a/source/extensions/filters/network/redis_proxy/proxy_filter.cc +++ b/source/extensions/filters/network/redis_proxy/proxy_filter.cc @@ -62,7 +62,8 @@ void ProxyFilter::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& ca void ProxyFilter::onRespValue(Common::Redis::RespValuePtr&& value) { pending_requests_.emplace_back(*this); PendingRequest& request = pending_requests_.back(); - CommandSplitter::SplitRequestPtr split = splitter_.makeRequest(std::move(value), request); + CommandSplitter::SplitRequestPtr split = + splitter_.makeRequest(std::move(value), request, callbacks_->connection().dispatcher()); if (split) { // The splitter can immediately respond and destroy the pending request. Only store the handle // if the request is still alive. diff --git a/source/extensions/filters/network/redis_proxy/proxy_filter.h b/source/extensions/filters/network/redis_proxy/proxy_filter.h index 1694a2a0640e..103e7c33c9e6 100644 --- a/source/extensions/filters/network/redis_proxy/proxy_filter.h +++ b/source/extensions/filters/network/redis_proxy/proxy_filter.h @@ -94,6 +94,8 @@ class ProxyFilter : public Network::ReadFilter, bool connectionAllowed() { return connection_allowed_; } private: + friend class RedisProxyFilterTest; + struct PendingRequest : public CommandSplitter::SplitCallbacks { PendingRequest(ProxyFilter& parent); ~PendingRequest() override; diff --git a/test/extensions/filters/network/common/redis/BUILD b/test/extensions/filters/network/common/redis/BUILD index dffc23954488..e8e445c8b608 100644 --- a/test/extensions/filters/network/common/redis/BUILD +++ b/test/extensions/filters/network/common/redis/BUILD @@ -60,3 +60,16 @@ envoy_cc_test( "@envoy_api//envoy/extensions/filters/network/redis_proxy/v3:pkg_cc_proto", ], ) + +envoy_cc_test( + name = "fault_test", + srcs = ["fault_test.cc"], + deps = [ + ":redis_mocks", + "//source/common/common:assert_lib", + "//source/extensions/filters/network/common/redis:fault_lib", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:test_runtime_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/network/common/redis/fault_test.cc b/test/extensions/filters/network/common/redis/fault_test.cc new file mode 100644 index 000000000000..a80caf5c2d2a --- /dev/null +++ b/test/extensions/filters/network/common/redis/fault_test.cc @@ -0,0 +1,206 @@ +#include "envoy/common/random_generator.h" + +#include "common/common/assert.h" + +#include "extensions/filters/network/common/redis/fault_impl.h" + +#include "test/extensions/filters/network/common/redis/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/printers.h" +#include "test/test_common/test_runtime.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + +using RedisProxy = envoy::extensions::filters::network::redis_proxy::v3::RedisProxy; +using FractionalPercent = envoy::type::v3::FractionalPercent; +class FaultTest : public testing::Test { +public: + const std::string RUNTIME_KEY = "runtime_key"; + + void + createCommandFault(RedisProxy::RedisFault* fault, std::string command_str, int delay_seconds, + absl::optional fault_percentage, + absl::optional denominator, + absl::optional runtime_key) { + // We don't set fault type as it isn't used in the test + + auto* commands = fault->mutable_commands(); + auto* command = commands->Add(); + command->assign(command_str); + + fault->set_fault_type(envoy::extensions::filters::network::redis_proxy::v3:: + RedisProxy_RedisFault_RedisFaultType_ERROR); + + addFaultPercentage(fault, fault_percentage, denominator, runtime_key); + addDelay(fault, delay_seconds); + } + + void + createAllKeyFault(RedisProxy::RedisFault* fault, int delay_seconds, + absl::optional fault_percentage, + absl::optional denominator, + absl::optional runtime_key) { + addFaultPercentage(fault, fault_percentage, denominator, runtime_key); + addDelay(fault, delay_seconds); + } + + void + addFaultPercentage(RedisProxy::RedisFault* fault, absl::optional fault_percentage, + absl::optional denominator, + absl::optional runtime_key) { + envoy::config::core::v3::RuntimeFractionalPercent* fault_enabled = + fault->mutable_fault_enabled(); + + if (runtime_key.has_value()) { + fault_enabled->set_runtime_key(runtime_key.value()); + } + auto* percentage = fault_enabled->mutable_default_value(); + if (fault_percentage.has_value()) { + percentage->set_numerator(fault_percentage.value()); + } + if (denominator.has_value()) { + percentage->set_denominator(denominator.value()); + } + } + + void addDelay(RedisProxy::RedisFault* fault, int delay_seconds) { + std::chrono::seconds duration = std::chrono::seconds(delay_seconds); + fault->mutable_delay()->set_seconds(duration.count()); + } + + testing::NiceMock random_; + testing::NiceMock runtime_; +}; + +TEST_F(FaultTest, MakeFaultForTestHelper) { + Common::Redis::FaultSharedPtr fault_ptr = + FaultManagerImpl::makeFaultForTest(FaultType::Error, std::chrono::milliseconds(10)); + + ASSERT_TRUE(fault_ptr->faultType() == FaultType::Error); + ASSERT_TRUE(fault_ptr->delayMs() == std::chrono::milliseconds(10)); +} + +TEST_F(FaultTest, NoFaults) { + RedisProxy redis_config; + auto* faults = redis_config.mutable_faults(); + + TestScopedRuntime scoped_runtime; + FaultManagerImpl fault_manager = FaultManagerImpl(random_, runtime_, *faults); + + const Fault* fault_ptr = fault_manager.getFaultForCommand("get"); + ASSERT_TRUE(fault_ptr == nullptr); +} + +TEST_F(FaultTest, SingleCommandFaultNotEnabled) { + RedisProxy redis_config; + auto* faults = redis_config.mutable_faults(); + createCommandFault(faults->Add(), "get", 0, 0, FractionalPercent::HUNDRED, RUNTIME_KEY); + + TestScopedRuntime scoped_runtime; + FaultManagerImpl fault_manager = FaultManagerImpl(random_, runtime_, *faults); + + EXPECT_CALL(random_, random()).WillOnce(Return(0)); + EXPECT_CALL(runtime_, snapshot()); + const Fault* fault_ptr = fault_manager.getFaultForCommand("get"); + ASSERT_TRUE(fault_ptr == nullptr); +} + +TEST_F(FaultTest, SingleCommandFault) { + // Inject a single fault. Notably we use a different denominator to test that code path; normally + // we use FractionalPercent::HUNDRED. + RedisProxy redis_config; + auto* faults = redis_config.mutable_faults(); + createCommandFault(faults->Add(), "ttl", 0, 5000, FractionalPercent::TEN_THOUSAND, RUNTIME_KEY); + + TestScopedRuntime scoped_runtime; + FaultManagerImpl fault_manager = FaultManagerImpl(random_, runtime_, *faults); + + EXPECT_CALL(random_, random()).WillOnce(Return(1)); + EXPECT_CALL(runtime_.snapshot_, getInteger(RUNTIME_KEY, 50)).WillOnce(Return(10)); + + const Fault* fault_ptr = fault_manager.getFaultForCommand("ttl"); + ASSERT_TRUE(fault_ptr != nullptr); +} + +TEST_F(FaultTest, SingleCommandFaultWithNoDefaultValueOrRuntimeValue) { + // Inject a single fault with no default value or runtime value. + RedisProxy redis_config; + auto* faults = redis_config.mutable_faults(); + createCommandFault(faults->Add(), "ttl", 0, absl::nullopt, absl::nullopt, absl::nullopt); + + TestScopedRuntime scoped_runtime; + FaultManagerImpl fault_manager = FaultManagerImpl(random_, runtime_, *faults); + + EXPECT_CALL(random_, random()).WillOnce(Return(1)); + const Fault* fault_ptr = fault_manager.getFaultForCommand("ttl"); + ASSERT_TRUE(fault_ptr == nullptr); +} + +TEST_F(FaultTest, MultipleFaults) { + // This creates 2 faults, but the map will have 3 entries, as each command points to + // command specific faults AND the general fault. The second fault has no runtime key, + // forcing the runtime key check to be false in application code and falling back to the + // default value. + RedisProxy redis_config; + auto* faults = redis_config.mutable_faults(); + createCommandFault(faults->Add(), "get", 0, 25, FractionalPercent::HUNDRED, RUNTIME_KEY); + createAllKeyFault(faults->Add(), 2, 25, FractionalPercent::HUNDRED, absl::nullopt); + + TestScopedRuntime scoped_runtime; + FaultManagerImpl fault_manager = FaultManagerImpl(random_, runtime_, *faults); + const Fault* fault_ptr; + + // Get command - should have a fault 50% of time + // For the first call we mock the random percentage to be 10%, which will give us the first fault + // with 0s delay. + EXPECT_CALL(random_, random()).WillOnce(Return(1)); + EXPECT_CALL(runtime_.snapshot_, getInteger(_, 25)).WillOnce(Return(10)); + fault_ptr = fault_manager.getFaultForCommand("get"); + ASSERT_TRUE(fault_ptr != nullptr); + ASSERT_EQ(fault_ptr->delayMs(), std::chrono::milliseconds(0)); + + // Another Get; we mock the random percentage to be 25%, giving us the ALL_KEY fault + EXPECT_CALL(random_, random()).WillOnce(Return(25)); + EXPECT_CALL(runtime_.snapshot_, getInteger(_, _)) + .Times(2) + .WillOnce(Return(10)) + .WillOnce(Return(50)); + fault_ptr = fault_manager.getFaultForCommand("get"); + ASSERT_TRUE(fault_ptr != nullptr); + ASSERT_EQ(fault_ptr->delayMs(), std::chrono::milliseconds(2000)); + + // No fault for Get command with mocked random percentage >= 50%. + EXPECT_CALL(random_, random()).WillOnce(Return(50)); + EXPECT_CALL(runtime_.snapshot_, getInteger(_, _)).Times(2); + fault_ptr = fault_manager.getFaultForCommand("get"); + ASSERT_TRUE(fault_ptr == nullptr); + + // Any other command; we mock the random percentage to be 1%, giving us the ALL_KEY fault + EXPECT_CALL(random_, random()).WillOnce(Return(1)); + EXPECT_CALL(runtime_.snapshot_, getInteger(_, _)).WillOnce(Return(10)); + + fault_ptr = fault_manager.getFaultForCommand("ttl"); + ASSERT_TRUE(fault_ptr != nullptr); + ASSERT_EQ(fault_ptr->delayMs(), std::chrono::milliseconds(2000)); + + // No fault for any other command with mocked random percentage >= 25%. + EXPECT_CALL(random_, random()).WillOnce(Return(25)); + EXPECT_CALL(runtime_.snapshot_, getInteger(_, _)); + fault_ptr = fault_manager.getFaultForCommand("ttl"); + ASSERT_TRUE(fault_ptr == nullptr); +} + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/redis_proxy/BUILD b/test/extensions/filters/network/redis_proxy/BUILD index 13980d9b57ca..034d9bd11b7c 100644 --- a/test/extensions/filters/network/redis_proxy/BUILD +++ b/test/extensions/filters/network/redis_proxy/BUILD @@ -18,16 +18,22 @@ envoy_extension_cc_test( name = "command_splitter_impl_test", srcs = ["command_splitter_impl_test.cc"], extension_name = "envoy.filters.network.redis_proxy", + # This test takes a while to run specially under tsan. + # Shard it to avoid test timeout. + shard_count = 2, deps = [ ":redis_mocks", "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", + "//source/extensions/filters/network/common/redis:fault_lib", "//source/extensions/filters/network/redis_proxy:command_splitter_lib", "//source/extensions/filters/network/redis_proxy:router_interface", "//test/extensions/filters/network/common/redis:redis_mocks", "//test/mocks:common_lib", + "//test/mocks/event:event_mocks", "//test/mocks/stats:stats_mocks", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", ], ) @@ -84,6 +90,7 @@ envoy_cc_mock( "//source/extensions/common/redis:cluster_refresh_manager_interface", "//source/extensions/filters/network/common/redis:client_interface", "//source/extensions/filters/network/common/redis:codec_lib", + "//source/extensions/filters/network/common/redis:fault_interface", "//source/extensions/filters/network/redis_proxy:command_splitter_interface", "//source/extensions/filters/network/redis_proxy:conn_pool_interface", "//source/extensions/filters/network/redis_proxy:router_interface", @@ -115,6 +122,7 @@ envoy_extension_cc_benchmark_binary( "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", "//source/extensions/filters/network/redis_proxy:command_splitter_lib", + "//test/mocks/event:event_mocks", "//test/test_common:printers_lib", "//test/test_common:simulated_time_system_lib", ], @@ -147,6 +155,7 @@ envoy_extension_cc_test( extension_name = "envoy.filters.network.redis_proxy", tags = ["fails_on_windows"], deps = [ + "//source/extensions/filters/network/common/redis:fault_lib", "//source/extensions/filters/network/redis_proxy:config", "//test/integration:integration_lib", ], diff --git a/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc b/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc index edf29c973092..3b6dee686f25 100644 --- a/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc +++ b/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc @@ -12,10 +12,14 @@ #include "extensions/filters/network/common/redis/supported_commands.h" #include "extensions/filters/network/redis_proxy/command_splitter_impl.h" +#include "test/extensions/filters/network/redis_proxy/mocks.h" +#include "test/mocks/event/mocks.h" #include "test/test_common/simulated_time_system.h" #include "benchmark/benchmark.h" +using testing::NiceMock; + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -54,21 +58,24 @@ class CommandLookUpSpeedTest { for (const std::string& command : Common::Redis::SupportedCommands::simpleCommands()) { Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {command, "hello"}); - splitter_.makeRequest(std::move(request), callbacks_); + splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); } for (const std::string& command : Common::Redis::SupportedCommands::evalCommands()) { Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {command, "hello"}); - splitter_.makeRequest(std::move(request), callbacks_); + splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); } } Router* router_{new NullRouterImpl()}; Stats::IsolatedStoreImpl store_; Event::SimulatedTimeSystem time_system_; - CommandSplitter::InstanceImpl splitter_{RouterPtr{router_}, store_, "redis.foo.", time_system_, - false}; + NiceMock fault_manager_; + NiceMock dispatcher_; + CommandSplitter::InstanceImpl splitter_{ + RouterPtr{router_}, store_, "redis.foo.", + time_system_, false, std::make_unique>(fault_manager_)}; NoOpSplitCallbacks callbacks_; CommandSplitter::SplitRequestPtr handle_; }; diff --git a/test/extensions/filters/network/redis_proxy/command_splitter_impl_test.cc b/test/extensions/filters/network/redis_proxy/command_splitter_impl_test.cc index 097cb3d49f4c..ad6526b5ecac 100644 --- a/test/extensions/filters/network/redis_proxy/command_splitter_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/command_splitter_impl_test.cc @@ -6,12 +6,14 @@ #include "common/common/fmt.h" #include "common/stats/isolated_store_impl.h" +#include "extensions/filters/network/common/redis/fault_impl.h" #include "extensions/filters/network/common/redis/supported_commands.h" #include "extensions/filters/network/redis_proxy/command_splitter_impl.h" #include "test/extensions/filters/network/common/redis/mocks.h" #include "test/extensions/filters/network/redis_proxy/mocks.h" #include "test/mocks/common.h" +#include "test/mocks/event/mocks.h" #include "test/mocks/stats/mocks.h" #include "test/test_common/simulated_time_system.h" @@ -32,7 +34,12 @@ namespace CommandSplitter { class RedisCommandSplitterImplTest : public testing::Test { public: RedisCommandSplitterImplTest() : RedisCommandSplitterImplTest(false) {} - RedisCommandSplitterImplTest(bool latency_in_macro) : latency_in_micros_(latency_in_macro) {} + RedisCommandSplitterImplTest(bool latency_in_macro) + : RedisCommandSplitterImplTest(latency_in_macro, nullptr) {} + RedisCommandSplitterImplTest(bool latency_in_macro, Common::Redis::FaultSharedPtr fault_ptr) + : latency_in_micros_(latency_in_macro) { + ON_CALL(*getFaultManager(), getFaultForCommand(_)).WillByDefault(Return(fault_ptr.get())); + } void makeBulkStringArray(Common::Redis::RespValue& value, const std::vector& strings) { std::vector values(strings.size()); @@ -50,6 +57,11 @@ class RedisCommandSplitterImplTest : public testing::Test { route_->policies_.push_back(mirror_policy); } + MockFaultManager* getFaultManager() { + auto fault_manager_ptr = splitter_.fault_manager_.get(); + return static_cast(fault_manager_ptr); + } + const bool latency_in_micros_; ConnPool::MockInstance* conn_pool_{new ConnPool::MockInstance()}; ConnPool::MockInstance* mirror_conn_pool_{new ConnPool::MockInstance()}; @@ -57,9 +69,16 @@ class RedisCommandSplitterImplTest : public testing::Test { std::shared_ptr> route_{ new NiceMock(ConnPool::InstanceSharedPtr{conn_pool_})}; NiceMock store_; + NiceMock dispatcher_; + NiceMock fault_manager_; + Event::SimulatedTimeSystem time_system_; - InstanceImpl splitter_{std::make_unique>(route_), store_, "redis.foo.", - time_system_, latency_in_micros_}; + InstanceImpl splitter_{std::make_unique>(route_), + store_, + "redis.foo.", + time_system_, + latency_in_micros_, + std::make_unique>(fault_manager_)}; MockSplitCallbacks callbacks_; SplitRequestPtr handle_; }; @@ -71,7 +90,7 @@ TEST_F(RedisCommandSplitterImplTest, AuthWithNoPassword) { EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {"auth"}); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); EXPECT_EQ(1UL, store_.counter("redis.foo.splitter.invalid_request").value()); } @@ -84,7 +103,7 @@ TEST_F(RedisCommandSplitterImplTest, CommandWhenAuthStillNeeded) { EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {"get", "foo"}); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); } TEST_F(RedisCommandSplitterImplTest, InvalidRequestNotArray) { @@ -93,7 +112,7 @@ TEST_F(RedisCommandSplitterImplTest, InvalidRequestNotArray) { response.asString() = Response::get().InvalidRequest; EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); EXPECT_EQ(1UL, store_.counter("redis.foo.splitter.invalid_request").value()); } @@ -105,7 +124,7 @@ TEST_F(RedisCommandSplitterImplTest, InvalidRequestEmptyArray) { EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; request->type(Common::Redis::RespType::Array); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); EXPECT_EQ(1UL, store_.counter("redis.foo.splitter.invalid_request").value()); } @@ -118,7 +137,7 @@ TEST_F(RedisCommandSplitterImplTest, InvalidRequestArrayTooSmall) { EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {"incr"}); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); EXPECT_EQ(1UL, store_.counter("redis.foo.splitter.invalid_request").value()); } @@ -131,7 +150,7 @@ TEST_F(RedisCommandSplitterImplTest, InvalidRequestArrayNotStrings) { Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {"incr", ""}); request->asArray()[1].type(Common::Redis::RespType::Null); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); EXPECT_EQ(1UL, store_.counter("redis.foo.splitter.invalid_request").value()); } @@ -144,7 +163,7 @@ TEST_F(RedisCommandSplitterImplTest, UnsupportedCommand) { EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {"newcommand", "hello"}); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); EXPECT_EQ(1UL, store_.counter("redis.foo.splitter.unsupported_command").value()); } @@ -172,7 +191,7 @@ class RedisSingleServerRequestTest : public RedisCommandSplitterImplTest, .WillOnce(DoAll(WithArg<2>(SaveArgAddress(&mirror_pool_callbacks_)), Return(&mirror_pool_request_))); } - handle_ = splitter_.makeRequest(std::move(request), callbacks_); + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); } void fail() { @@ -343,7 +362,7 @@ TEST_P(RedisSingleServerRequestTest, NoUpstream) { response.type(Common::Redis::RespType::Error); response.asString() = Response::get().NoUpstreamHost; EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); - handle_ = splitter_.makeRequest(std::move(request), callbacks_); + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); EXPECT_EQ(nullptr, handle_); }; @@ -365,7 +384,7 @@ TEST_F(RedisSingleServerRequestTest, PingSuccess) { EXPECT_CALL(callbacks_, connectionAllowed()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); - handle_ = splitter_.makeRequest(std::move(request), callbacks_); + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); EXPECT_EQ(nullptr, handle_); }; @@ -425,13 +444,13 @@ TEST_F(RedisSingleServerRequestTest, EvalWrongNumberOfArgs) { EXPECT_CALL(callbacks_, connectionAllowed()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); makeBulkStringArray(*request1, {"eval", "return {ARGV[1]}"}); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request1), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request1), callbacks_, dispatcher_)); response.asString() = "wrong number of arguments for 'evalsha' command"; EXPECT_CALL(callbacks_, connectionAllowed()).WillOnce(Return(true)); EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); makeBulkStringArray(*request2, {"evalsha", "return {ARGV[1]}", "1"}); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request2), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request2), callbacks_, dispatcher_)); }; TEST_F(RedisSingleServerRequestTest, EvalNoUpstream) { @@ -447,7 +466,7 @@ TEST_F(RedisSingleServerRequestTest, EvalNoUpstream) { response.type(Common::Redis::RespType::Error); response.asString() = Response::get().NoUpstreamHost; EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); - handle_ = splitter_.makeRequest(std::move(request), callbacks_); + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); EXPECT_EQ(nullptr, handle_); EXPECT_EQ(1UL, store_.counter("redis.foo.command.eval.total").value()); @@ -508,7 +527,7 @@ class FragmentedRequestCommandHandlerTest : public RedisCommandSplitterImplTest } } - handle_ = splitter_.makeRequest(std::move(request), callbacks_); + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); } std::vector> expected_requests_; @@ -846,7 +865,7 @@ TEST_F(RedisMSETCommandHandlerTest, WrongNumberOfArgs) { EXPECT_CALL(callbacks_, onResponse_(PointeesEq(&response))); Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; makeBulkStringArray(*request, {"mset", "foo", "bar", "fizz"}); - EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_)); + EXPECT_EQ(nullptr, splitter_.makeRequest(std::move(request), callbacks_, dispatcher_)); EXPECT_EQ(1UL, store_.counter("redis.foo.command.mset.total").value()); EXPECT_EQ(1UL, store_.counter("redis.foo.command.mset.error").value()); }; @@ -993,6 +1012,147 @@ INSTANTIATE_TEST_SUITE_P(RedisSingleServerRequestWithLatencyMicrosTest, RedisSingleServerRequestWithLatencyMicrosTest, testing::ValuesIn(Common::Redis::SupportedCommands::simpleCommands())); +// In subclasses of fault test, we mock the expected faults in the constructor, as the +// fault manager is owned by the splitter, which is also generated later in construction +// of the base test class. +class RedisSingleServerRequestWithFaultTest : public RedisSingleServerRequestTest { +public: + NiceMock* timer_; + Event::TimerCb timer_cb_; + int delay_ms_; + Common::Redis::FaultSharedPtr fault_ptr_; +}; + +class RedisSingleServerRequestWithErrorFaultTest : public RedisSingleServerRequestWithFaultTest { +public: + RedisSingleServerRequestWithErrorFaultTest() { + delay_ms_ = 0; + fault_ptr_ = Common::Redis::FaultManagerImpl::makeFaultForTest( + Common::Redis::FaultType::Error, std::chrono::milliseconds(delay_ms_)); + ON_CALL(*getFaultManager(), getFaultForCommand(_)).WillByDefault(Return(fault_ptr_.get())); + } +}; + +TEST_P(RedisSingleServerRequestWithErrorFaultTest, Fault) { + InSequence s; + + std::string lower_command = absl::AsciiStrToLower(GetParam()); + Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; + makeBulkStringArray(*request, {GetParam(), "hello"}); + + EXPECT_CALL(callbacks_, connectionAllowed()).WillOnce(Return(true)); + EXPECT_CALL(callbacks_, onResponse_(_)); + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); + EXPECT_EQ(nullptr, handle_); + + EXPECT_EQ(1UL, store_.counter(fmt::format("redis.foo.command.{}.total", lower_command)).value()); + EXPECT_EQ(1UL, store_.counter(fmt::format("redis.foo.command.{}.error", lower_command)).value()); + EXPECT_EQ(1UL, + store_.counter(fmt::format("redis.foo.command.{}.error_fault", lower_command)).value()); +}; + +class RedisSingleServerRequestWithErrorWithDelayFaultTest + : public RedisSingleServerRequestWithFaultTest { +public: + RedisSingleServerRequestWithErrorWithDelayFaultTest() { + delay_ms_ = 5; + fault_ptr_ = Common::Redis::FaultManagerImpl::makeFaultForTest( + Common::Redis::FaultType::Error, std::chrono::milliseconds(delay_ms_)); + ON_CALL(*getFaultManager(), getFaultForCommand(_)).WillByDefault(Return(fault_ptr_.get())); + timer_ = new NiceMock(); + } +}; + +INSTANTIATE_TEST_SUITE_P(RedisSingleServerRequestWithErrorFaultTest, + RedisSingleServerRequestWithErrorFaultTest, + testing::ValuesIn(Common::Redis::SupportedCommands::simpleCommands())); + +TEST_P(RedisSingleServerRequestWithErrorWithDelayFaultTest, Fault) { + InSequence s; + + std::string lower_command = absl::AsciiStrToLower(GetParam()); + Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; + makeBulkStringArray(*request, {GetParam(), "hello"}); + + // As error faults have zero latency, recorded latency is equal to the delay. + EXPECT_CALL(callbacks_, connectionAllowed()).WillOnce(Return(true)); + EXPECT_CALL(dispatcher_, createTimer_(_)).WillOnce(Invoke([this](Event::TimerCb timer_cb) { + timer_cb_ = timer_cb; + return timer_; + })); + + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); + EXPECT_NE(nullptr, handle_); + time_system_.setMonotonicTime(std::chrono::milliseconds(delay_ms_)); + EXPECT_CALL(store_, deliverHistogramToSinks( + Property(&Stats::Metric::name, + fmt::format("redis.foo.command.{}.latency", lower_command)), + delay_ms_)); + EXPECT_CALL(callbacks_, onResponse_(_)); + timer_cb_(); + + EXPECT_EQ(1UL, store_.counter(fmt::format("redis.foo.command.{}.total", lower_command)).value()); + EXPECT_EQ(1UL, store_.counter(fmt::format("redis.foo.command.{}.error", lower_command)).value()); + EXPECT_EQ(1UL, + store_.counter(fmt::format("redis.foo.command.{}.error_fault", lower_command)).value()); +}; + +INSTANTIATE_TEST_SUITE_P(RedisSingleServerRequestWithErrorWithDelayFaultTest, + RedisSingleServerRequestWithErrorWithDelayFaultTest, + testing::ValuesIn(Common::Redis::SupportedCommands::simpleCommands())); + +class RedisSingleServerRequestWithDelayFaultTest : public RedisSingleServerRequestWithFaultTest { +public: + RedisSingleServerRequestWithDelayFaultTest() { + delay_ms_ = 15; + fault_ptr_ = Common::Redis::FaultManagerImpl::makeFaultForTest( + Common::Redis::FaultType::Delay, std::chrono::milliseconds(delay_ms_)); + ON_CALL(*getFaultManager(), getFaultForCommand(_)).WillByDefault(Return(fault_ptr_.get())); + timer_ = new NiceMock(); + } +}; + +TEST_P(RedisSingleServerRequestWithDelayFaultTest, Fault) { + InSequence s; + + std::string lower_command = absl::AsciiStrToLower(GetParam()); + std::string hash_key = "hello"; + + Common::Redis::RespValuePtr request{new Common::Redis::RespValue()}; + makeBulkStringArray(*request, {GetParam(), "hello"}); + + EXPECT_CALL(callbacks_, connectionAllowed()).WillOnce(Return(true)); + EXPECT_CALL(dispatcher_, createTimer_(_)).WillOnce(Invoke([this](Event::TimerCb timer_cb) { + timer_cb_ = timer_cb; + return timer_; + })); + EXPECT_CALL(*conn_pool_, makeRequest_(hash_key, RespVariantEq(*request), _)) + .WillOnce(DoAll(WithArg<2>(SaveArgAddress(&pool_callbacks_)), Return(&pool_request_))); + + handle_ = splitter_.makeRequest(std::move(request), callbacks_, dispatcher_); + + EXPECT_NE(nullptr, handle_); + + EXPECT_CALL(store_, deliverHistogramToSinks( + Property(&Stats::Metric::name, + fmt::format("redis.foo.command.{}.latency", lower_command)), + delay_ms_)); + respond(); + + time_system_.setMonotonicTime(std::chrono::milliseconds(delay_ms_)); + timer_cb_(); + + EXPECT_EQ(1UL, store_.counter(fmt::format("redis.foo.command.{}.total", lower_command)).value()); + EXPECT_EQ(1UL, + store_.counter(fmt::format("redis.foo.command.{}.success", lower_command)).value()); + EXPECT_EQ(1UL, + store_.counter(fmt::format("redis.foo.command.{}.delay_fault", lower_command)).value()); +}; + +INSTANTIATE_TEST_SUITE_P(RedisSingleServerRequestWithDelayFaultTest, + RedisSingleServerRequestWithDelayFaultTest, + testing::ValuesIn(Common::Redis::SupportedCommands::simpleCommands())); + } // namespace CommandSplitter } // namespace RedisProxy } // namespace NetworkFilters diff --git a/test/extensions/filters/network/redis_proxy/config_test.cc b/test/extensions/filters/network/redis_proxy/config_test.cc index a9043af8cd6e..155b72689284 100644 --- a/test/extensions/filters/network/redis_proxy/config_test.cc +++ b/test/extensions/filters/network/redis_proxy/config_test.cc @@ -169,6 +169,43 @@ stat_prefix: foo cb(connection); } +TEST(RedisProxyFilterConfigFactoryTest, RedisProxyFaultProto) { + const std::string yaml = R"EOF( +prefix_routes: + catch_all_route: + cluster: fake_cluster +stat_prefix: foo +faults: +- fault_type: ERROR + fault_enabled: + default_value: + numerator: 30 + denominator: HUNDRED + runtime_key: "bogus_key" + commands: + - GET +- fault_type: DELAY + fault_enabled: + default_value: + numerator: 20 + denominator: HUNDRED + runtime_key: "bogus_key" + delay: 2s +settings: + op_timeout: 0.02s + )EOF"; + + envoy::extensions::filters::network::redis_proxy::v3::RedisProxy proto_config{}; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + NiceMock context; + RedisProxyFilterConfigFactory factory; + Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); + EXPECT_TRUE(factory.isTerminalFilter()); + Network::MockConnection connection; + EXPECT_CALL(connection, addReadFilter(_)); + cb(connection); +} + // Test that the deprecated extension name still functions. TEST(RedisProxyFilterConfigFactoryTest, DEPRECATED_FEATURE_TEST(DeprecatedExtensionFilterName)) { const std::string deprecated_name = "envoy.redis_proxy"; diff --git a/test/extensions/filters/network/redis_proxy/mocks.cc b/test/extensions/filters/network/redis_proxy/mocks.cc index d51809ba27f6..b5190823ab8d 100644 --- a/test/extensions/filters/network/redis_proxy/mocks.cc +++ b/test/extensions/filters/network/redis_proxy/mocks.cc @@ -26,6 +26,10 @@ MockMirrorPolicy::MockMirrorPolicy(ConnPool::InstanceSharedPtr conn_pool) ON_CALL(*this, shouldMirror(_)).WillByDefault(Return(true)); } +MockFaultManager::MockFaultManager() = default; +MockFaultManager::MockFaultManager(const MockFaultManager&) {} +MockFaultManager::~MockFaultManager() = default; + namespace ConnPool { MockPoolCallbacks::MockPoolCallbacks() = default; diff --git a/test/extensions/filters/network/redis_proxy/mocks.h b/test/extensions/filters/network/redis_proxy/mocks.h index b093ad35b9b9..e90842bba06d 100644 --- a/test/extensions/filters/network/redis_proxy/mocks.h +++ b/test/extensions/filters/network/redis_proxy/mocks.h @@ -7,6 +7,7 @@ #include "extensions/common/redis/cluster_refresh_manager.h" #include "extensions/filters/network/common/redis/client.h" #include "extensions/filters/network/common/redis/codec_impl.h" +#include "extensions/filters/network/common/redis/fault.h" #include "extensions/filters/network/redis_proxy/command_splitter.h" #include "extensions/filters/network/redis_proxy/conn_pool.h" #include "extensions/filters/network/redis_proxy/router.h" @@ -51,6 +52,15 @@ class MockMirrorPolicy : public MirrorPolicy { ConnPool::InstanceSharedPtr conn_pool_; }; +class MockFaultManager : public Common::Redis::FaultManager { +public: + MockFaultManager(); + MockFaultManager(const MockFaultManager& other); + ~MockFaultManager() override; + + MOCK_METHOD(const Common::Redis::Fault*, getFaultForCommand, (const std::string&), (const)); +}; + namespace ConnPool { class MockPoolCallbacks : public PoolCallbacks { @@ -110,13 +120,14 @@ class MockInstance : public Instance { MockInstance(); ~MockInstance() override; - SplitRequestPtr makeRequest(Common::Redis::RespValuePtr&& request, - SplitCallbacks& callbacks) override { - return SplitRequestPtr{makeRequest_(*request, callbacks)}; + SplitRequestPtr makeRequest(Common::Redis::RespValuePtr&& request, SplitCallbacks& callbacks, + Event::Dispatcher& dispatcher) override { + return SplitRequestPtr{makeRequest_(*request, callbacks, dispatcher)}; } MOCK_METHOD(SplitRequest*, makeRequest_, - (const Common::Redis::RespValue& request, SplitCallbacks& callbacks)); + (const Common::Redis::RespValue& request, SplitCallbacks& callbacks, + Event::Dispatcher& dispatcher)); }; } // namespace CommandSplitter diff --git a/test/extensions/filters/network/redis_proxy/proxy_filter_test.cc b/test/extensions/filters/network/redis_proxy/proxy_filter_test.cc index f094c02b665a..3e9335c4c5a2 100644 --- a/test/extensions/filters/network/redis_proxy/proxy_filter_test.cc +++ b/test/extensions/filters/network/redis_proxy/proxy_filter_test.cc @@ -171,25 +171,34 @@ class RedisProxyFilterTest : public testing::Test, public Common::Redis::Decoder NiceMock api_; }; -TEST_F(RedisProxyFilterTest, OutOfOrderResponseWithDrainClose) { - InSequence s; +class RedisProxyFilterTestWithTwoCallbacks : public RedisProxyFilterTest { +public: + CommandSplitter::MockSplitRequest* request_handle1_{new CommandSplitter::MockSplitRequest()}; + CommandSplitter::MockSplitRequest* request_handle2_{new CommandSplitter::MockSplitRequest()}; + CommandSplitter::SplitCallbacks* request_callbacks1_; + CommandSplitter::SplitCallbacks* request_callbacks2_; - Buffer::OwnedImpl fake_data; - CommandSplitter::MockSplitRequest* request_handle1 = new CommandSplitter::MockSplitRequest(); - CommandSplitter::SplitCallbacks* request_callbacks1; - CommandSplitter::MockSplitRequest* request_handle2 = new CommandSplitter::MockSplitRequest(); - CommandSplitter::SplitCallbacks* request_callbacks2; - EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { + void decodeHelper(Buffer::Instance&) { Common::Redis::RespValuePtr request1(new Common::Redis::RespValue()); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request1), _)) - .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&request_callbacks1)), Return(request_handle1))); + EXPECT_CALL(splitter_, makeRequest_(Ref(*request1), _, _)) + .WillOnce( + DoAll(WithArg<1>(SaveArgAddress(&request_callbacks1_)), Return(request_handle1_))); decoder_callbacks_->onRespValue(std::move(request1)); Common::Redis::RespValuePtr request2(new Common::Redis::RespValue()); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request2), _)) - .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&request_callbacks2)), Return(request_handle2))); + EXPECT_CALL(splitter_, makeRequest_(Ref(*request2), _, _)) + .WillOnce( + DoAll(WithArg<1>(SaveArgAddress(&request_callbacks2_)), Return(request_handle2_))); decoder_callbacks_->onRespValue(std::move(request2)); - })); + } +}; + +TEST_F(RedisProxyFilterTestWithTwoCallbacks, OutOfOrderResponseWithDrainClose) { + InSequence s; + + Buffer::OwnedImpl fake_data; + EXPECT_CALL(*decoder_, decode(Ref(fake_data))) + .WillOnce(Invoke(this, &RedisProxyFilterTestWithTwoCallbacks::decodeHelper)); EXPECT_EQ(Network::FilterStatus::Continue, filter_->onData(fake_data, false)); EXPECT_EQ(2UL, config_->stats_.downstream_rq_total_.value()); @@ -197,7 +206,7 @@ TEST_F(RedisProxyFilterTest, OutOfOrderResponseWithDrainClose) { Common::Redis::RespValuePtr response2(new Common::Redis::RespValue()); Common::Redis::RespValue* response2_ptr = response2.get(); - request_callbacks2->onResponse(std::move(response2)); + request_callbacks2_->onResponse(std::move(response2)); Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(*encoder_, encode(Ref(*response1), _)); @@ -207,38 +216,25 @@ TEST_F(RedisProxyFilterTest, OutOfOrderResponseWithDrainClose) { EXPECT_CALL(runtime_.snapshot_, featureEnabled("redis.drain_close_enabled", 100)) .WillOnce(Return(true)); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); - request_callbacks1->onResponse(std::move(response1)); + request_callbacks1_->onResponse(std::move(response1)); EXPECT_EQ(1UL, config_->stats_.downstream_cx_drain_close_.value()); } -TEST_F(RedisProxyFilterTest, OutOfOrderResponseDownstreamDisconnectBeforeFlush) { +TEST_F(RedisProxyFilterTestWithTwoCallbacks, OutOfOrderResponseDownstreamDisconnectBeforeFlush) { InSequence s; Buffer::OwnedImpl fake_data; - CommandSplitter::MockSplitRequest* request_handle1 = new CommandSplitter::MockSplitRequest(); - CommandSplitter::SplitCallbacks* request_callbacks1; - CommandSplitter::MockSplitRequest* request_handle2 = new CommandSplitter::MockSplitRequest(); - CommandSplitter::SplitCallbacks* request_callbacks2; - EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { - Common::Redis::RespValuePtr request1(new Common::Redis::RespValue()); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request1), _)) - .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&request_callbacks1)), Return(request_handle1))); - decoder_callbacks_->onRespValue(std::move(request1)); - - Common::Redis::RespValuePtr request2(new Common::Redis::RespValue()); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request2), _)) - .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&request_callbacks2)), Return(request_handle2))); - decoder_callbacks_->onRespValue(std::move(request2)); - })); + EXPECT_CALL(*decoder_, decode(Ref(fake_data))) + .WillOnce(Invoke(this, &RedisProxyFilterTestWithTwoCallbacks::decodeHelper)); EXPECT_EQ(Network::FilterStatus::Continue, filter_->onData(fake_data, false)); EXPECT_EQ(2UL, config_->stats_.downstream_rq_total_.value()); EXPECT_EQ(2UL, config_->stats_.downstream_rq_active_.value()); Common::Redis::RespValuePtr response2(new Common::Redis::RespValue()); - request_callbacks2->onResponse(std::move(response2)); - EXPECT_CALL(*request_handle1, cancel()); + request_callbacks2_->onResponse(std::move(response2)); + EXPECT_CALL(*request_handle1_, cancel()); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } @@ -251,7 +247,7 @@ TEST_F(RedisProxyFilterTest, DownstreamDisconnectWithActive) { CommandSplitter::SplitCallbacks* request_callbacks1; EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { Common::Redis::RespValuePtr request1(new Common::Redis::RespValue()); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request1), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request1), _, _)) .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&request_callbacks1)), Return(request_handle1))); decoder_callbacks_->onRespValue(std::move(request1)); })); @@ -269,10 +265,10 @@ TEST_F(RedisProxyFilterTest, ImmediateResponse) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request1)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request1), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request1), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { Common::Redis::RespValuePtr error(new Common::Redis::RespValue()); error->type(Common::Redis::RespType::Error); error->asString() = "no healthy upstream"; @@ -313,10 +309,10 @@ TEST_F(RedisProxyFilterTest, AuthWhenNotRequired) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { EXPECT_TRUE(callbacks.connectionAllowed()); Common::Redis::RespValuePtr error(new Common::Redis::RespValue()); error->type(Common::Redis::RespType::Error); @@ -340,10 +336,10 @@ TEST_F(RedisProxyFilterTest, AuthAclWhenNotRequired) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { EXPECT_TRUE(callbacks.connectionAllowed()); Common::Redis::RespValuePtr error(new Common::Redis::RespValue()); error->type(Common::Redis::RespType::Error); @@ -383,10 +379,10 @@ TEST_F(RedisProxyFilterWithAuthPasswordTest, AuthPasswordCorrect) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { EXPECT_FALSE(callbacks.connectionAllowed()); Common::Redis::RespValuePtr reply(new Common::Redis::RespValue()); reply->type(Common::Redis::RespType::SimpleString); @@ -410,10 +406,10 @@ TEST_F(RedisProxyFilterWithAuthPasswordTest, AuthPasswordIncorrect) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { EXPECT_FALSE(callbacks.connectionAllowed()); Common::Redis::RespValuePtr reply(new Common::Redis::RespValue()); reply->type(Common::Redis::RespType::Error); @@ -455,10 +451,10 @@ TEST_F(RedisProxyFilterWithAuthAclTest, AuthAclCorrect) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { EXPECT_FALSE(callbacks.connectionAllowed()); Common::Redis::RespValuePtr reply(new Common::Redis::RespValue()); reply->type(Common::Redis::RespType::SimpleString); @@ -482,10 +478,10 @@ TEST_F(RedisProxyFilterWithAuthAclTest, AuthAclUsernameIncorrect) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { EXPECT_FALSE(callbacks.connectionAllowed()); Common::Redis::RespValuePtr reply(new Common::Redis::RespValue()); reply->type(Common::Redis::RespType::Error); @@ -509,10 +505,10 @@ TEST_F(RedisProxyFilterWithAuthAclTest, AuthAclPasswordIncorrect) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { decoder_callbacks_->onRespValue(std::move(request)); })); - EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _)) + EXPECT_CALL(splitter_, makeRequest_(Ref(*request), _, _)) .WillOnce( - Invoke([&](const Common::Redis::RespValue&, - CommandSplitter::SplitCallbacks& callbacks) -> CommandSplitter::SplitRequest* { + Invoke([&](const Common::Redis::RespValue&, CommandSplitter::SplitCallbacks& callbacks, + Event::Dispatcher&) -> CommandSplitter::SplitRequest* { EXPECT_FALSE(callbacks.connectionAllowed()); Common::Redis::RespValuePtr reply(new Common::Redis::RespValue()); reply->type(Common::Redis::RespType::Error); diff --git a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc index 8d9a09c6b464..78d87cbd6096 100644 --- a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc +++ b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc @@ -1,6 +1,9 @@ #include #include +#include "common/common/fmt.h" + +#include "extensions/filters/network/common/redis/fault_impl.h" #include "extensions/filters/network/redis_proxy/command_splitter_impl.h" #include "test/integration/integration.h" @@ -52,7 +55,7 @@ const std::string CONFIG = fmt::format(R"EOF( filters: name: redis typed_config: - "@type": type.googleapis.com/envoy.config.filter.network.redis_proxy.v2.RedisProxy + "@type": type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy stat_prefix: redis_stats prefix_routes: catch_all_route: @@ -277,6 +280,27 @@ const std::string CONFIG_WITH_ROUTES_AND_AUTH_PASSWORDS = )EOF", TestEnvironment::nullDevicePath()); +// This is a configuration with fault injection enabled. +const std::string CONFIG_WITH_FAULT_INJECTION = CONFIG + R"EOF( + faults: + - fault_type: ERROR + fault_enabled: + default_value: + numerator: 100 + denominator: HUNDRED + commands: + - GET + - fault_type: DELAY + fault_enabled: + default_value: + numerator: 20 + denominator: HUNDRED + runtime_key: "bogus_key" + delay: 2s + commands: + - SET +)EOF"; + // This function encodes commands as an array of bulkstrings as transmitted by Redis clients to // Redis servers, according to the Redis protocol. std::string makeBulkStringArray(std::vector&& command_strings) { @@ -440,6 +464,12 @@ class RedisProxyWithCommandStatsIntegrationTest : public RedisProxyIntegrationTe : RedisProxyIntegrationTest(CONFIG_WITH_COMMAND_STATS, 2) {} }; +class RedisProxyWithFaultInjectionIntegrationTest : public RedisProxyIntegrationTest { +public: + RedisProxyWithFaultInjectionIntegrationTest() + : RedisProxyIntegrationTest(CONFIG_WITH_FAULT_INJECTION, 2) {} +}; + INSTANTIATE_TEST_SUITE_P(IpVersions, RedisProxyIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); @@ -472,6 +502,10 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, RedisProxyWithCommandStatsIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); +INSTANTIATE_TEST_SUITE_P(IpVersions, RedisProxyWithFaultInjectionIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + void RedisProxyIntegrationTest::initialize() { setUpstreamCount(num_upstreams_); setDeterministic(); @@ -1066,5 +1100,25 @@ TEST_P(RedisProxyWithMirrorsIntegrationTest, EnabledViaRuntimeFraction) { redis_client->close(); } +TEST_P(RedisProxyWithFaultInjectionIntegrationTest, ErrorFault) { + std::string fault_response = + fmt::format("-{}\r\n", Extensions::NetworkFilters::Common::Redis::FaultMessages::get().Error); + initialize(); + simpleProxyResponse(makeBulkStringArray({"get", "foo"}), fault_response); + + EXPECT_EQ(1, test_server_->counter("redis.redis_stats.command.get.error")->value()); + EXPECT_EQ(1, test_server_->counter("redis.redis_stats.command.get.error_fault")->value()); +} + +TEST_P(RedisProxyWithFaultInjectionIntegrationTest, DelayFault) { + const std::string& set_request = makeBulkStringArray({"set", "write_only:toto", "bar"}); + const std::string& set_response = ":1\r\n"; + initialize(); + simpleRequestAndResponse(set_request, set_response); + + EXPECT_EQ(1, test_server_->counter("redis.redis_stats.command.set.success")->value()); + EXPECT_EQ(1, test_server_->counter("redis.redis_stats.command.set.delay_fault")->value()); +} + } // namespace } // namespace Envoy From 90a97c7ef3d67e213980db6b0ebeff284a938837 Mon Sep 17 00:00:00 2001 From: Yutong Li Date: Wed, 12 Aug 2020 11:14:09 -0700 Subject: [PATCH 47/67] api: add node field to csds request (#12522) Added an node field to csds request to identify the CSDS client to the CSDS server, and removed the [#not-implemented-hide:] for the endpoint_config since it has been implemented in #11577 Risk Level: Low Testing: N/A Docs Changes: N/A Release Notes: N/A Signed-off-by: Yutong Li --- api/envoy/service/status/v3/csds.proto | 4 +++- api/envoy/service/status/v4alpha/csds.proto | 4 +++- generated_api_shadow/envoy/service/status/v3/csds.proto | 4 +++- generated_api_shadow/envoy/service/status/v4alpha/csds.proto | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/api/envoy/service/status/v3/csds.proto b/api/envoy/service/status/v3/csds.proto index beccfb8cb58e..23f1352bf489 100644 --- a/api/envoy/service/status/v3/csds.proto +++ b/api/envoy/service/status/v3/csds.proto @@ -61,6 +61,9 @@ message ClientStatusRequest { // Management server can use these match criteria to identify clients. // The match follows OR semantics. repeated type.matcher.v3.NodeMatcher node_matchers = 1; + + // The node making the csds request. + config.core.v3.Node node = 2; } // Detailed config (per xDS) with status. @@ -80,7 +83,6 @@ message PerXdsConfig { admin.v3.ScopedRoutesConfigDump scoped_route_config = 5; - // [#not-implemented-hide:] admin.v3.EndpointsConfigDump endpoint_config = 6; } } diff --git a/api/envoy/service/status/v4alpha/csds.proto b/api/envoy/service/status/v4alpha/csds.proto index 2286eb94a8a7..37758954cadb 100644 --- a/api/envoy/service/status/v4alpha/csds.proto +++ b/api/envoy/service/status/v4alpha/csds.proto @@ -61,6 +61,9 @@ message ClientStatusRequest { // Management server can use these match criteria to identify clients. // The match follows OR semantics. repeated type.matcher.v4alpha.NodeMatcher node_matchers = 1; + + // The node making the csds request. + config.core.v4alpha.Node node = 2; } // Detailed config (per xDS) with status. @@ -80,7 +83,6 @@ message PerXdsConfig { admin.v4alpha.ScopedRoutesConfigDump scoped_route_config = 5; - // [#not-implemented-hide:] admin.v4alpha.EndpointsConfigDump endpoint_config = 6; } } diff --git a/generated_api_shadow/envoy/service/status/v3/csds.proto b/generated_api_shadow/envoy/service/status/v3/csds.proto index beccfb8cb58e..23f1352bf489 100644 --- a/generated_api_shadow/envoy/service/status/v3/csds.proto +++ b/generated_api_shadow/envoy/service/status/v3/csds.proto @@ -61,6 +61,9 @@ message ClientStatusRequest { // Management server can use these match criteria to identify clients. // The match follows OR semantics. repeated type.matcher.v3.NodeMatcher node_matchers = 1; + + // The node making the csds request. + config.core.v3.Node node = 2; } // Detailed config (per xDS) with status. @@ -80,7 +83,6 @@ message PerXdsConfig { admin.v3.ScopedRoutesConfigDump scoped_route_config = 5; - // [#not-implemented-hide:] admin.v3.EndpointsConfigDump endpoint_config = 6; } } diff --git a/generated_api_shadow/envoy/service/status/v4alpha/csds.proto b/generated_api_shadow/envoy/service/status/v4alpha/csds.proto index 2286eb94a8a7..37758954cadb 100644 --- a/generated_api_shadow/envoy/service/status/v4alpha/csds.proto +++ b/generated_api_shadow/envoy/service/status/v4alpha/csds.proto @@ -61,6 +61,9 @@ message ClientStatusRequest { // Management server can use these match criteria to identify clients. // The match follows OR semantics. repeated type.matcher.v4alpha.NodeMatcher node_matchers = 1; + + // The node making the csds request. + config.core.v4alpha.Node node = 2; } // Detailed config (per xDS) with status. @@ -80,7 +83,6 @@ message PerXdsConfig { admin.v4alpha.ScopedRoutesConfigDump scoped_route_config = 5; - // [#not-implemented-hide:] admin.v4alpha.EndpointsConfigDump endpoint_config = 6; } } From 7f3e8a85ff1dbc3f3aec54dad7b5450aa53a98cd Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Wed, 12 Aug 2020 15:38:11 -0700 Subject: [PATCH 48/67] caching: Improved the tests and coverage of the CacheFilter tree (#12544) Signed-off-by: Yosry Ahmed --- .../filters/http/cache/cache_filter.cc | 1 + .../filters/http/cache/cache_headers_utils.cc | 25 - .../filters/http/cache/cache_headers_utils.h | 3 - .../filters/http/cache/cacheability_utils.cc | 6 +- .../filters/http/cache/http_cache.cc | 22 - .../filters/http/cache/http_cache.h | 4 - test/extensions/filters/http/cache/BUILD | 16 +- .../cache/cache_filter_integration_test.cc | 253 +++--- .../filters/http/cache/cache_filter_test.cc | 729 ++++++++++++++++-- .../http/cache/cache_headers_utils_test.cc | 9 +- .../http/cache/cacheability_utils_test.cc | 156 ++-- test/extensions/filters/http/cache/common.h | 134 ++++ .../filters/http/cache/config_test.cc | 1 + .../filters/http/cache/http_cache_test.cc | 342 +++++--- .../http/cache/simple_http_cache/BUILD | 1 + .../simple_http_cache_test.cc | 1 + tools/spelling/spelling_dictionary.txt | 1 + 17 files changed, 1287 insertions(+), 417 deletions(-) create mode 100644 test/extensions/filters/http/cache/common.h diff --git a/source/extensions/filters/http/cache/cache_filter.cc b/source/extensions/filters/http/cache/cache_filter.cc index 18ab3e625c26..007a2a6b6010 100644 --- a/source/extensions/filters/http/cache/cache_filter.cc +++ b/source/extensions/filters/http/cache/cache_filter.cc @@ -262,6 +262,7 @@ void CacheFilter::processSuccessfulValidation(Http::ResponseHeaderMap& response_ } } +// TODO(yosrym93): Write a test that exercises this when SimpleHttpCache implements updateHeaders bool CacheFilter::shouldUpdateCachedEntry(const Http::ResponseHeaderMap& response_headers) const { ASSERT(isResponseNotModified(response_headers), "shouldUpdateCachedEntry must only be called with 304 responses"); diff --git a/source/extensions/filters/http/cache/cache_headers_utils.cc b/source/extensions/filters/http/cache/cache_headers_utils.cc index 27d08bde0088..7d6559104258 100644 --- a/source/extensions/filters/http/cache/cache_headers_utils.cc +++ b/source/extensions/filters/http/cache/cache_headers_utils.cc @@ -106,31 +106,6 @@ ResponseCacheControl::ResponseCacheControl(absl::string_view cache_control_heade } } -std::ostream& operator<<(std::ostream& os, const OptionalDuration& duration) { - return duration.has_value() ? os << duration.value().count() : os << " "; -} - -std::ostream& operator<<(std::ostream& os, const RequestCacheControl& request_cache_control) { - return os << "{" - << "must_validate: " << request_cache_control.must_validate_ << ", " - << "no_store: " << request_cache_control.no_store_ << ", " - << "no_transform: " << request_cache_control.no_transform_ << ", " - << "only_if_cached: " << request_cache_control.only_if_cached_ << ", " - << "max_age: " << request_cache_control.max_age_ << ", " - << "min_fresh: " << request_cache_control.min_fresh_ << ", " - << "max_stale: " << request_cache_control.max_stale_ << "}"; -} - -std::ostream& operator<<(std::ostream& os, const ResponseCacheControl& response_cache_control) { - return os << "{" - << "must_validate: " << response_cache_control.must_validate_ << ", " - << "no_store: " << response_cache_control.no_store_ << ", " - << "no_transform: " << response_cache_control.no_transform_ << ", " - << "no_stale: " << response_cache_control.no_stale_ << ", " - << "public: " << response_cache_control.is_public_ << ", " - << "max_age: " << response_cache_control.max_age_ << "}"; -} - bool operator==(const RequestCacheControl& lhs, const RequestCacheControl& rhs) { return (lhs.must_validate_ == rhs.must_validate_) && (lhs.no_store_ == rhs.no_store_) && (lhs.no_transform_ == rhs.no_transform_) && (lhs.only_if_cached_ == rhs.only_if_cached_) && diff --git a/source/extensions/filters/http/cache/cache_headers_utils.h b/source/extensions/filters/http/cache/cache_headers_utils.h index 8a185d88b40d..79193ce4f21b 100644 --- a/source/extensions/filters/http/cache/cache_headers_utils.h +++ b/source/extensions/filters/http/cache/cache_headers_utils.h @@ -82,9 +82,6 @@ struct ResponseCacheControl { OptionalDuration max_age_; }; -std::ostream& operator<<(std::ostream& os, const OptionalDuration& duration); -std::ostream& operator<<(std::ostream& os, const RequestCacheControl& request_cache_control); -std::ostream& operator<<(std::ostream& os, const ResponseCacheControl& response_cache_control); bool operator==(const RequestCacheControl& lhs, const RequestCacheControl& rhs); bool operator==(const ResponseCacheControl& lhs, const ResponseCacheControl& rhs); diff --git a/source/extensions/filters/http/cache/cacheability_utils.cc b/source/extensions/filters/http/cache/cacheability_utils.cc index 784c37953c9d..029569521c85 100644 --- a/source/extensions/filters/http/cache/cacheability_utils.cc +++ b/source/extensions/filters/http/cache/cacheability_utils.cc @@ -61,9 +61,11 @@ bool CacheabilityUtils::isCacheableResponse(const Http::ResponseHeaderMap& heade ResponseCacheControl response_cache_control(cache_control); // Only cache responses with explicit validation data, either: - // max-age or s-maxage cache-control directives with date header. - // expires header. + // "no-cache" cache-control directive + // "max-age" or "s-maxage" cache-control directives with date header + // expires header const bool has_validation_data = + response_cache_control.must_validate_ || (headers.Date() && response_cache_control.max_age_.has_value()) || headers.get(Http::Headers::get().Expires); diff --git a/source/extensions/filters/http/cache/http_cache.cc b/source/extensions/filters/http/cache/http_cache.cc index 43943444c897..236f20d3d619 100644 --- a/source/extensions/filters/http/cache/http_cache.cc +++ b/source/extensions/filters/http/cache/http_cache.cc @@ -22,28 +22,6 @@ namespace Extensions { namespace HttpFilters { namespace Cache { -std::ostream& operator<<(std::ostream& os, CacheEntryStatus status) { - switch (status) { - case CacheEntryStatus::Ok: - return os << "Ok"; - case CacheEntryStatus::Unusable: - return os << "Unusable"; - case CacheEntryStatus::RequiresValidation: - return os << "RequiresValidation"; - case CacheEntryStatus::FoundNotModified: - return os << "FoundNotModified"; - case CacheEntryStatus::NotSatisfiableRange: - return os << "NotSatisfiableRange"; - case CacheEntryStatus::SatisfiableRange: - return os << "SatisfiableRange"; - } - NOT_REACHED_GCOVR_EXCL_LINE; -} - -std::ostream& operator<<(std::ostream& os, const AdjustedByteRange& range) { - return os << "[" << range.begin() << "," << range.end() << ")"; -} - LookupRequest::LookupRequest(const Http::RequestHeaderMap& request_headers, SystemTime timestamp) : timestamp_(timestamp) { // These ASSERTs check prerequisites. A request without these headers can't be looked up in cache; diff --git a/source/extensions/filters/http/cache/http_cache.h b/source/extensions/filters/http/cache/http_cache.h index 907a8d02be96..9a2736994ea6 100644 --- a/source/extensions/filters/http/cache/http_cache.h +++ b/source/extensions/filters/http/cache/http_cache.h @@ -41,8 +41,6 @@ enum class CacheEntryStatus { SatisfiableRange, }; -std::ostream& operator<<(std::ostream& os, CacheEntryStatus status); - // Byte range from an HTTP request. class RawByteRange { public: @@ -112,8 +110,6 @@ inline bool operator==(const AdjustedByteRange& lhs, const AdjustedByteRange& rh return lhs.begin() == rhs.begin() && lhs.end() == rhs.end(); } -std::ostream& operator<<(std::ostream& os, const AdjustedByteRange& range); - // Adjusts request_range_spec to fit a cached response of size content_length, putting the results // in response_ranges. Returns true if response_ranges is satisfiable (empty is considered // satisfiable, as it denotes the entire body). diff --git a/test/extensions/filters/http/cache/BUILD b/test/extensions/filters/http/cache/BUILD index db5d5ea50fd5..c17f76a3a206 100644 --- a/test/extensions/filters/http/cache/BUILD +++ b/test/extensions/filters/http/cache/BUILD @@ -1,4 +1,4 @@ -load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//bazel:envoy_build_system.bzl", "envoy_cc_test_library", "envoy_package") load( "//test/extensions:extensions_build_system.bzl", "envoy_extension_cc_test", @@ -8,11 +8,22 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_test_library( + name = "common", + hdrs = ["common.h"], + deps = [ + "//source/extensions/filters/http/cache:cache_headers_utils_lib", + "//source/extensions/filters/http/cache:http_cache_lib", + "//source/extensions/filters/http/cache/simple_http_cache:simple_http_cache_lib", + ], +) + envoy_extension_cc_test( name = "cache_headers_utils_test", srcs = ["cache_headers_utils_test.cc"], extension_name = "envoy.filters.http.cache", deps = [ + ":common", "//include/envoy/http:header_map_interface", "//source/common/http:header_map_lib", "//source/extensions/filters/http/cache:cache_headers_utils_lib", @@ -25,6 +36,7 @@ envoy_extension_cc_test( srcs = ["http_cache_test.cc"], extension_name = "envoy.filters.http.cache", deps = [ + ":common", "//source/extensions/filters/http/cache:http_cache_lib", "//source/extensions/filters/http/cache/simple_http_cache:simple_http_cache_lib", "//test/mocks/http:http_mocks", @@ -38,6 +50,7 @@ envoy_extension_cc_test( srcs = ["cache_filter_test.cc"], extension_name = "envoy.filters.http.cache", deps = [ + ":common", "//source/extensions/filters/http/cache:cache_filter_lib", "//source/extensions/filters/http/cache/simple_http_cache:simple_http_cache_lib", "//test/mocks/server:factory_context_mocks", @@ -75,7 +88,6 @@ envoy_extension_cc_test( "cache_filter_integration_test.cc", ], extension_name = "envoy.filters.http.cache", - tags = ["fails_on_windows"], deps = [ "//source/extensions/filters/http/cache:config", "//source/extensions/filters/http/cache:http_cache_lib", diff --git a/test/extensions/filters/http/cache/cache_filter_integration_test.cc b/test/extensions/filters/http/cache/cache_filter_integration_test.cc index d4113c78c7a8..fde9cd63fe4c 100644 --- a/test/extensions/filters/http/cache/cache_filter_integration_test.cc +++ b/test/extensions/filters/http/cache/cache_filter_integration_test.cc @@ -5,6 +5,7 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Cache { +namespace { // TODO(toddmgreer): Expand integration test to include age header values, // expiration, range headers, HEAD requests, trailers, config customizations, @@ -51,45 +52,51 @@ TEST_P(CacheIntegrationTest, MissInsertHit) { {":path", absl::StrCat("/", protocolTestParamsToString({GetParam(), 0}))}, {":scheme", "http"}, {":authority", "MissInsertHit"}}; - Http::TestResponseHeaderMapImpl response_headers = {{":status", "200"}, - {"date", formatter_.now(simTime())}, - {"cache-control", "public,max-age=3600"}, - {"content-length", "42"}}; + + const std::string response_body(42, 'a'); + Http::TestResponseHeaderMapImpl response_headers = { + {":status", "200"}, + {"date", formatter_.now(simTime())}, + {"cache-control", "public,max-age=3600"}, + {"content-length", std::to_string(response_body.size())}}; // Send first request, and get response from upstream. { - IntegrationStreamDecoderPtr request = codec_client_->makeHeaderOnlyRequest(request_headers); + IntegrationStreamDecoderPtr response_decoder = + codec_client_->makeHeaderOnlyRequest(request_headers); waitForNextUpstreamRequest(); upstream_request_->encodeHeaders(response_headers, /*end_stream=*/false); // send 42 'a's - upstream_request_->encodeData(42, true); + upstream_request_->encodeData(response_body, /*end_stream=*/true); // Wait for the response to be read by the codec client. - request->waitForEndStream(); - EXPECT_TRUE(request->complete()); - EXPECT_THAT(request->headers(), IsSupersetOfHeaders(response_headers)); - EXPECT_EQ(request->headers().get(Http::Headers::get().Age), nullptr); - EXPECT_EQ(request->body(), std::string(42, 'a')); - EXPECT_EQ(waitForAccessLog(access_log_name_), - fmt::format("- via_upstream{}", TestEnvironment::newLine)); + response_decoder->waitForEndStream(); + EXPECT_TRUE(response_decoder->complete()); + EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(response_headers)); + EXPECT_EQ(response_decoder->headers().get(Http::Headers::get().Age), nullptr); + EXPECT_EQ(response_decoder->body(), response_body); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::HasSubstr("- via_upstream")); } // Advance time, to verify the original date header is preserved. simTime().advanceTimeWait(std::chrono::seconds(10)); // Send second request, and get response from cache. - IntegrationStreamDecoderPtr request = codec_client_->makeHeaderOnlyRequest(request_headers); - request->waitForEndStream(); - EXPECT_TRUE(request->complete()); - EXPECT_THAT(request->headers(), IsSupersetOfHeaders(response_headers)); - EXPECT_EQ(request->body(), std::string(42, 'a')); - EXPECT_NE(request->headers().get(Http::Headers::get().Age), nullptr); - // Advance time to force a log flush. - simTime().advanceTimeWait(std::chrono::seconds(1)); - EXPECT_EQ(waitForAccessLog(access_log_name_, 1), - fmt::format("RFCF cache.response_from_cache_filter{}", TestEnvironment::newLine)); + { + IntegrationStreamDecoderPtr response_decoder = + codec_client_->makeHeaderOnlyRequest(request_headers); + response_decoder->waitForEndStream(); + EXPECT_TRUE(response_decoder->complete()); + EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(response_headers)); + EXPECT_EQ(response_decoder->body(), response_body); + EXPECT_NE(response_decoder->headers().get(Http::Headers::get().Age), nullptr); + // Advance time to force a log flush. + simTime().advanceTimeWait(std::chrono::seconds(1)); + EXPECT_THAT(waitForAccessLog(access_log_name_, 1), + testing::HasSubstr("RFCF cache.response_from_cache_filter")); + } } -TEST_P(CacheIntegrationTest, SuccessfulValidation) { +TEST_P(CacheIntegrationTest, ExpiredValidated) { useAccessLog("%RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS%"); // Set system time to cause Envoy's cached formatted time to match time on this thread. simTime().setSystemTime(std::chrono::hours(1)); @@ -100,14 +107,15 @@ TEST_P(CacheIntegrationTest, SuccessfulValidation) { {":method", "GET"}, {":path", absl::StrCat("/", protocolTestParamsToString({GetParam(), 0}))}, {":scheme", "http"}, - {":authority", "SuccessfulValidation"}}; + {":authority", "ExpiredValidated"}}; - const std::string original_response_date = formatter_.now(simTime()); - Http::TestResponseHeaderMapImpl response_headers = {{":status", "200"}, - {"date", original_response_date}, - {"cache-control", "max-age=0"}, - {"content-length", "42"}, - {"etag", "abc123"}}; + const std::string response_body(42, 'a'); + Http::TestResponseHeaderMapImpl response_headers = { + {":status", "200"}, + {"date", formatter_.now(simTime())}, + {"cache-control", "max-age=10"}, // expires after 10 s + {"content-length", std::to_string(response_body.size())}, + {"etag", "abc123"}}; // Send first request, and get response from upstream. { @@ -116,54 +124,57 @@ TEST_P(CacheIntegrationTest, SuccessfulValidation) { waitForNextUpstreamRequest(); upstream_request_->encodeHeaders(response_headers, /*end_stream=*/false); // send 42 'a's - upstream_request_->encodeData(42, true); + upstream_request_->encodeData(response_body, true); // Wait for the response to be read by the codec client. response_decoder->waitForEndStream(); EXPECT_TRUE(response_decoder->complete()); EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(response_headers)); EXPECT_EQ(response_decoder->headers().get(Http::Headers::get().Age), nullptr); - EXPECT_EQ(response_decoder->body(), std::string(42, 'a')); - EXPECT_EQ(waitForAccessLog(access_log_name_), "- via_upstream\n"); + EXPECT_EQ(response_decoder->body(), response_body); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::HasSubstr("- via_upstream")); } - simTime().advanceTimeWait(std::chrono::seconds(10)); - const std::string not_modified_date = formatter_.now(simTime()); - - // Send second request, the cached response should be validated then served. - IntegrationStreamDecoderPtr response_decoder = - codec_client_->makeHeaderOnlyRequest(request_headers); - waitForNextUpstreamRequest(); - - // Check for injected conditional headers -- no "Last-Modified" header so should fallback to - // "Date". - Http::TestRequestHeaderMapImpl injected_headers = {{"if-none-match", "abc123"}, - {"if-modified-since", original_response_date}}; - EXPECT_THAT(upstream_request_->headers(), IsSupersetOfHeaders(injected_headers)); - - // Create a 304 (not modified) response -> cached response is valid. - Http::TestResponseHeaderMapImpl not_modified_response_headers = {{":status", "304"}, - {"date", not_modified_date}}; - upstream_request_->encodeHeaders(not_modified_response_headers, /*end_stream=*/true); - - // The original response headers should be updated with 304 response headers. - response_headers.setDate(not_modified_date); - - // Wait for the response to be read by the codec client. - response_decoder->waitForEndStream(); - - // Check that the served response is the cached response. - EXPECT_TRUE(response_decoder->complete()); - EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(response_headers)); - EXPECT_EQ(response_decoder->body(), std::string(42, 'a')); - // Check that age header exists as this is a cached response. - EXPECT_NE(response_decoder->headers().get(Http::Headers::get().Age), nullptr); - - // Advance time to force a log flush. - simTime().advanceTimeWait(std::chrono::seconds(1)); - EXPECT_EQ(waitForAccessLog(access_log_name_, 1), "RFCF cache.response_from_cache_filter\n"); + // Advance time for the cached response to be stale (expired) + // Also to make sure response date header gets updated with the 304 date + simTime().advanceTimeWait(std::chrono::seconds(11)); + + // Send second request, the cached response should be validate then served + { + IntegrationStreamDecoderPtr response_decoder = + codec_client_->makeHeaderOnlyRequest(request_headers); + waitForNextUpstreamRequest(); + + // Check for injected precondition headers + const Http::TestRequestHeaderMapImpl injected_headers = {{"if-none-match", "abc123"}}; + EXPECT_THAT(upstream_request_->headers(), IsSupersetOfHeaders(injected_headers)); + + // Create a 304 (not modified) response -> cached response is valid + const std::string not_modified_date = formatter_.now(simTime()); + const Http::TestResponseHeaderMapImpl not_modified_response_headers = { + {":status", "304"}, {"date", not_modified_date}}; + upstream_request_->encodeHeaders(not_modified_response_headers, /*end_stream=*/true); + + // The original response headers should be updated with 304 response headers + response_headers.setDate(not_modified_date); + + // Wait for the response to be read by the codec client. + response_decoder->waitForEndStream(); + + // Check that the served response is the cached response + EXPECT_TRUE(response_decoder->complete()); + EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(response_headers)); + EXPECT_EQ(response_decoder->body(), response_body); + // Check that age header exists as this is a cached response + EXPECT_NE(response_decoder->headers().get(Http::Headers::get().Age), nullptr); + + // Advance time to force a log flush. + simTime().advanceTimeWait(std::chrono::seconds(1)); + EXPECT_THAT(waitForAccessLog(access_log_name_, 1), + testing::HasSubstr("RFCF cache.response_from_cache_filter")); + } } -TEST_P(CacheIntegrationTest, UnsuccessfulValidation) { +TEST_P(CacheIntegrationTest, ExpiredFetchedNewResponse) { useAccessLog("%RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS%"); // Set system time to cause Envoy's cached formatted time to match time on this thread. simTime().setSystemTime(std::chrono::hours(1)); @@ -174,65 +185,73 @@ TEST_P(CacheIntegrationTest, UnsuccessfulValidation) { {":method", "GET"}, {":path", absl::StrCat("/", protocolTestParamsToString({GetParam(), 0}))}, {":scheme", "http"}, - {":authority", "UnsuccessfulValidation"}}; - - Http::TestResponseHeaderMapImpl original_response_headers = {{":status", "200"}, - {"date", formatter_.now(simTime())}, - {"cache-control", "max-age=0"}, - {"content-length", "10"}, - {"etag", "a1"}}; + {":authority", "ExpiredFetchedNewResponse"}}; // Send first request, and get response from upstream. { + const std::string response_body(10, 'a'); + Http::TestResponseHeaderMapImpl response_headers = { + {":status", "200"}, + {"date", formatter_.now(simTime())}, + {"cache-control", "max-age=10"}, // expires after 10 s + {"content-length", std::to_string(response_body.size())}, + {"etag", "a1"}}; + IntegrationStreamDecoderPtr response_decoder = codec_client_->makeHeaderOnlyRequest(request_headers); waitForNextUpstreamRequest(); - upstream_request_->encodeHeaders(original_response_headers, /*end_stream=*/false); + upstream_request_->encodeHeaders(response_headers, /*end_stream=*/false); // send 10 'a's - upstream_request_->encodeData(10, true); + upstream_request_->encodeData(response_body, /*end_stream=*/true); // Wait for the response to be read by the codec client. response_decoder->waitForEndStream(); EXPECT_TRUE(response_decoder->complete()); - EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(original_response_headers)); + EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(response_headers)); EXPECT_EQ(response_decoder->headers().get(Http::Headers::get().Age), nullptr); - EXPECT_EQ(response_decoder->body(), std::string(10, 'a')); - EXPECT_EQ(waitForAccessLog(access_log_name_), "- via_upstream\n"); + EXPECT_EQ(response_decoder->body(), response_body); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::HasSubstr("- via_upstream")); } - simTime().advanceTimeWait(std::chrono::seconds(10)); - // Any response with status other than 304 should be passed to the client as-is. - Http::TestResponseHeaderMapImpl updated_response_headers = {{":status", "200"}, - {"date", formatter_.now(simTime())}, - {"cache-control", "max-age=0"}, - {"content-length", "20"}, - {"etag", "a2"}}; - - // Send second request, validation of the cached response should be attempted but should fail. - IntegrationStreamDecoderPtr response_decoder = - codec_client_->makeHeaderOnlyRequest(request_headers); - waitForNextUpstreamRequest(); - - // Check for injected precondition headers. - Http::TestRequestHeaderMapImpl injected_headers = {{"if-none-match", "a1"}}; - EXPECT_THAT(upstream_request_->headers(), IsSupersetOfHeaders(injected_headers)); - - // Reply with the updated response -> cached response is invalid. - upstream_request_->encodeHeaders(updated_response_headers, /*end_stream=*/false); - // send 20 'a's - upstream_request_->encodeData(20, true); - - // Wait for the response to be read by the codec client. - response_decoder->waitForEndStream(); - // Check that the served response is the updated response. - EXPECT_TRUE(response_decoder->complete()); - EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(updated_response_headers)); - EXPECT_EQ(response_decoder->body(), std::string(20, 'a')); - // Check that age header does not exist as this is not a cached response. - EXPECT_EQ(response_decoder->headers().get(Http::Headers::get().Age), nullptr); - - // Advance time to force a log flush. - simTime().advanceTimeWait(std::chrono::seconds(1)); - EXPECT_EQ(waitForAccessLog(access_log_name_, 1), "- via_upstream\n"); + // Advance time for the cached response to be stale (expired) + // Also to make sure response date header gets updated with the 304 date + simTime().advanceTimeWait(std::chrono::seconds(11)); + + // Send second request, validation of the cached response should be attempted but should fail + // The new response should be served + { + const std::string response_body(20, 'a'); + Http::TestResponseHeaderMapImpl response_headers = { + {":status", "200"}, + {"date", formatter_.now(simTime())}, + {"content-length", std::to_string(response_body.size())}, + {"etag", "a2"}}; + + IntegrationStreamDecoderPtr response_decoder = + codec_client_->makeHeaderOnlyRequest(request_headers); + waitForNextUpstreamRequest(); + + // Check for injected precondition headers + Http::TestRequestHeaderMapImpl injected_headers = {{"if-none-match", "a1"}}; + EXPECT_THAT(upstream_request_->headers(), IsSupersetOfHeaders(injected_headers)); + + // Reply with the updated response -> cached response is invalid + upstream_request_->encodeHeaders(response_headers, /*end_stream=*/false); + // send 20 'a's + upstream_request_->encodeData(response_body, /*end_stream=*/true); + + // Wait for the response to be read by the codec client. + response_decoder->waitForEndStream(); + // Check that the served response is the updated response + EXPECT_TRUE(response_decoder->complete()); + EXPECT_THAT(response_decoder->headers(), IsSupersetOfHeaders(response_headers)); + EXPECT_EQ(response_decoder->body(), response_body); + // Check that age header does not exist as this is not a cached response + EXPECT_EQ(response_decoder->headers().get(Http::Headers::get().Age), nullptr); + + // Advance time to force a log flush. + simTime().advanceTimeWait(std::chrono::seconds(1)); + EXPECT_THAT(waitForAccessLog(access_log_name_), testing::HasSubstr("- via_upstream")); + } } // Send the same GET request with body and trailers twice, then check that the response @@ -273,6 +292,8 @@ TEST_P(CacheIntegrationTest, GetRequestWithBodyAndTrailers) { EXPECT_EQ(response->body(), std::string(42, 'a')); } } + +} // namespace } // namespace Cache } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/cache/cache_filter_test.cc b/test/extensions/filters/http/cache/cache_filter_test.cc index 0ab4034cc799..e64998e7c227 100644 --- a/test/extensions/filters/http/cache/cache_filter_test.cc +++ b/test/extensions/filters/http/cache/cache_filter_test.cc @@ -1,6 +1,11 @@ +#include "envoy/http/header_map.h" + +#include "common/http/headers.h" + #include "extensions/filters/http/cache/cache_filter.h" #include "extensions/filters/http/cache/simple_http_cache/simple_http_cache.h" +#include "test/extensions/filters/http/cache/common.h" #include "test/mocks/server/factory_context.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -13,41 +18,6 @@ namespace HttpFilters { namespace Cache { namespace { -// Wrapper for SimpleHttpCache that delays the onHeaders callback from getHeaders, for verifying -// that CacheFilter works correctly whether the onHeaders call happens immediately, or after -// getHeaders and decodeHeaders return. -class DelayedCache : public SimpleHttpCache { -public: - // HttpCache - LookupContextPtr makeLookupContext(LookupRequest&& request) override { - return std::make_unique( - SimpleHttpCache::makeLookupContext(std::move(request)), delayed_cb_); - } - InsertContextPtr makeInsertContext(LookupContextPtr&& lookup_context) override { - return SimpleHttpCache::makeInsertContext( - std::move(dynamic_cast(*lookup_context).context_)); - } - - std::function delayed_cb_; - -private: - class DelayedLookupContext : public LookupContext { - public: - DelayedLookupContext(LookupContextPtr&& context, std::function& delayed_cb) - : context_(std::move(context)), delayed_cb_(delayed_cb) {} - void getHeaders(LookupHeadersCallback&& cb) override { - delayed_cb_ = [this, cb]() mutable { context_->getHeaders(std::move(cb)); }; - } - void getBody(const AdjustedByteRange& range, LookupBodyCallback&& cb) override { - context_->getBody(range, std::move(cb)); - } - void getTrailers(LookupTrailersCallback&& cb) override { context_->getTrailers(std::move(cb)); } - - LookupContextPtr context_; - std::function& delayed_cb_; - }; -}; - class CacheFilterTest : public ::testing::Test { protected: CacheFilter makeFilter(HttpCache& cache) { @@ -73,17 +43,117 @@ class CacheFilterTest : public ::testing::Test { NiceMock encoder_callbacks_; }; +TEST_F(CacheFilterTest, UncacheableRequest) { + request_headers_.setHost("UncacheableRequest"); + + // POST requests are uncacheable + request_headers_.setMethod(Http::Headers::get().MethodValues.Post); + + for (int i = 0; i < 2; i++) { + // Create filter for request i + CacheFilter filter = makeFilter(simple_cache_); + + // Decode request i header + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // In the first request was cached the second request would return StopAllIterationAndWatermark + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode response header + EXPECT_EQ(filter.encodeHeaders(response_headers_, true), Http::FilterHeadersStatus::Continue); + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, UncacheableResponse) { + request_headers_.setHost("UncacheableResponse"); + + // Responses with "Cache-Control: no-store" are uncacheable + response_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-store"); + + for (int i = 0; i < 2; i++) { + // Create filter for request i + CacheFilter filter = makeFilter(simple_cache_); + + // Decode request i header + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // In the first request was cached the second request would return StopAllIterationAndWatermark + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode response header + EXPECT_EQ(filter.encodeHeaders(response_headers_, true), Http::FilterHeadersStatus::Continue); + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, ImmediateMiss) { + for (int request = 1; request <= 2; request++) { + // Each iteration a request is sent to a different host, therefore the second one is a miss + request_headers_.setHost("ImmediateMiss" + std::to_string(request)); + + // Create filter for request 1 + CacheFilter filter = makeFilter(simple_cache_); + + // Decode request 1 header + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode response header + EXPECT_EQ(filter.encodeHeaders(response_headers_, true), Http::FilterHeadersStatus::Continue); + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, DelayedMiss) { + for (int request = 1; request <= 2; request++) { + // Each iteration a request is sent to a different host, therefore the second one is a miss + request_headers_.setHost("DelayedMiss" + std::to_string(request)); + + // Create filter for request 1 + CacheFilter filter = makeFilter(delayed_cache_); + + // Decode request 1 header + // No hit will be found, so the filter should call continueDecoding + EXPECT_CALL(decoder_callbacks_, continueDecoding); + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // The filter should stop iteration waiting for cache response + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + + // Make the delayed callback to call onHeaders + delayed_cache_.delayed_headers_cb_(); + + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode response header + EXPECT_EQ(filter.encodeHeaders(response_headers_, true), Http::FilterHeadersStatus::Continue); + filter.onDestroy(); + } +} + TEST_F(CacheFilterTest, ImmediateHitNoBody) { request_headers_.setHost("ImmediateHitNoBody"); - ON_CALL(decoder_callbacks_, dispatcher()).WillByDefault(ReturnRef(context_.dispatcher_)); - ON_CALL(context_.dispatcher_, post(_)).WillByDefault(::testing::InvokeArgument<0>()); { // Create filter for request 1. CacheFilter filter = makeFilter(simple_cache_); - // Decode request 1 header. + // Decode request 1 header + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); // Encode response header. EXPECT_EQ(filter.encodeHeaders(response_headers_, true), Http::FilterHeadersStatus::Continue); @@ -96,8 +166,12 @@ TEST_F(CacheFilterTest, ImmediateHitNoBody) { // Decode request 2 header. EXPECT_CALL(decoder_callbacks_, encodeHeaders_(testing::AllOf(IsSupersetOfHeaders(response_headers_), - HeaderHasValueRef("age", "0")), + HeaderHasValueRef(Http::Headers::get().Age, "0")), true)); + // Make sure the filter did not encode any data + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // Make sure decoding does not continue + EXPECT_CALL(decoder_callbacks_, continueDecoding).Times(0); EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::StopAllIterationAndWatermark); ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); @@ -106,19 +180,25 @@ TEST_F(CacheFilterTest, ImmediateHitNoBody) { } TEST_F(CacheFilterTest, DelayedHitNoBody) { - request_headers_.setHost("ImmediateHitNoBody"); - ON_CALL(decoder_callbacks_, dispatcher()).WillByDefault(ReturnRef(context_.dispatcher_)); - ON_CALL(context_.dispatcher_, post(_)).WillByDefault(::testing::InvokeArgument<0>()); + request_headers_.setHost("DelayedHitNoBody"); { // Create filter for request 1. CacheFilter filter = makeFilter(delayed_cache_); - // Decode request 1 header. + // Decode request 1 header + // No hit will be found, so the filter should call continueDecoding + EXPECT_CALL(decoder_callbacks_, continueDecoding); + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // The filter should stop iteration waiting for cache response EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::StopAllIterationAndWatermark); - EXPECT_CALL(decoder_callbacks_, continueDecoding); - delayed_cache_.delayed_cb_(); + + // Make the delayed callback to call onHeaders + delayed_cache_.delayed_headers_cb_(); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); // Encode response header. @@ -129,14 +209,18 @@ TEST_F(CacheFilterTest, DelayedHitNoBody) { // Create filter for request 2. CacheFilter filter = makeFilter(delayed_cache_); - // Decode request 2 header. - EXPECT_EQ(filter.decodeHeaders(request_headers_, true), - Http::FilterHeadersStatus::StopAllIterationAndWatermark); + // Decode request 2 header EXPECT_CALL(decoder_callbacks_, encodeHeaders_(testing::AllOf(IsSupersetOfHeaders(response_headers_), - HeaderHasValueRef("age", "0")), + HeaderHasValueRef(Http::Headers::get().Age, "0")), true)); - delayed_cache_.delayed_cb_(); + // Make sure the filter did not encode any data + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // Make sure decoding does not continue + EXPECT_CALL(decoder_callbacks_, continueDecoding).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + delayed_cache_.delayed_headers_cb_(); ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); filter.onDestroy(); } @@ -144,15 +228,16 @@ TEST_F(CacheFilterTest, DelayedHitNoBody) { TEST_F(CacheFilterTest, ImmediateHitBody) { request_headers_.setHost("ImmediateHitBody"); - ON_CALL(decoder_callbacks_, dispatcher()).WillByDefault(ReturnRef(context_.dispatcher_)); - ON_CALL(context_.dispatcher_, post(_)).WillByDefault(::testing::InvokeArgument<0>()); const std::string body = "abc"; { // Create filter for request 1. CacheFilter filter = makeFilter(simple_cache_); - // Decode request 1 header. + // Decode request 1 header + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); // Encode response header. @@ -169,14 +254,403 @@ TEST_F(CacheFilterTest, ImmediateHitBody) { // Decode request 2 header EXPECT_CALL(decoder_callbacks_, encodeHeaders_(testing::AllOf(IsSupersetOfHeaders(response_headers_), - HeaderHasValueRef("age", "0")), + HeaderHasValueRef(Http::Headers::get().Age, "0")), + false)); + EXPECT_CALL( + decoder_callbacks_, + encodeData(testing::Property(&Buffer::Instance::toString, testing::Eq(body)), true)); + // Make sure decoding does not continue + EXPECT_CALL(decoder_callbacks_, continueDecoding).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, DelayedHitBody) { + request_headers_.setHost("DelayedHitBody"); + const std::string body = "abc"; + + { + // Create filter for request 1 + CacheFilter filter = makeFilter(delayed_cache_); + + // Decode request 1 header + // No hit will be found, so the filter should call continueDecoding + EXPECT_CALL(decoder_callbacks_, continueDecoding); + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // The filter should stop iteration waiting for cache response + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + + // Make the delayed callback to call onHeaders + delayed_cache_.delayed_headers_cb_(); + + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode response header + Buffer::OwnedImpl buffer(body); + response_headers_.setContentLength(body.size()); + EXPECT_EQ(filter.encodeHeaders(response_headers_, false), Http::FilterHeadersStatus::Continue); + EXPECT_EQ(filter.encodeData(buffer, true), Http::FilterDataStatus::Continue); + filter.onDestroy(); + } + { + // Create filter for request 2 + CacheFilter filter = makeFilter(delayed_cache_); + + // Decode request 2 header + + EXPECT_CALL(decoder_callbacks_, + encodeHeaders_(testing::AllOf(IsSupersetOfHeaders(response_headers_), + HeaderHasValueRef(Http::Headers::get().Age, "0")), false)); EXPECT_CALL( decoder_callbacks_, encodeData(testing::Property(&Buffer::Instance::toString, testing::Eq(body)), true)); + + // Make sure decoding does not continue + EXPECT_CALL(decoder_callbacks_, continueDecoding).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + + // Make the delayed callbacks to call onHeaders & onBody + delayed_cache_.delayed_headers_cb_(); + delayed_cache_.delayed_body_cb_(); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, ImmediateSuccessfulValidation) { + request_headers_.setHost("ImmediateSuccessfulValidation"); + const std::string body = "abc"; + + { + // Create filter for request 1 + CacheFilter filter = makeFilter(simple_cache_); + + // Decode request 1 header + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + + // Encode response + + // Add Etag & Last-Modified headers to the response for validation + response_headers_.setReferenceKey(Http::CustomHeaders::get().Etag, "abc123"); + response_headers_.setReferenceKey(Http::CustomHeaders::get().LastModified, + formatter_.now(time_source_)); + + Buffer::OwnedImpl buffer(body); + response_headers_.setContentLength(body.size()); + EXPECT_EQ(filter.encodeHeaders(response_headers_, false), Http::FilterHeadersStatus::Continue); + EXPECT_EQ(filter.encodeData(buffer, true), Http::FilterDataStatus::Continue); + filter.onDestroy(); + } + { + // Create filter for request 2 + CacheFilter filter = makeFilter(simple_cache_); + + // Make request require validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + + // Decode request 2 header + // Make sure the filter did not encode any headers or data (during decoding) + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + + // Make sure validation conditional headers are added + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-none-match", "abc123"}, {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode 304 response + // Advance time to make sure the cached date is updated with the 304 date + time_source_.advanceTimeWait(std::chrono::seconds(10)); + const std::string not_modified_date = formatter_.now(time_source_); + Http::TestResponseHeaderMapImpl not_modified_response_headers = {{":status", "304"}, + {"date", not_modified_date}}; + + // Check that the cached response body is encoded + Buffer::OwnedImpl buffer(body); + EXPECT_CALL( + encoder_callbacks_, + addEncodedData(testing::Property(&Buffer::Instance::toString, testing::Eq(body)), true)); + + // The cache is immediate so everything should be done before encodeHeaders returns + EXPECT_EQ(filter.encodeHeaders(not_modified_response_headers, true), + Http::FilterHeadersStatus::Continue); + + // Check for the cached response headers with updated date + Http::TestResponseHeaderMapImpl updated_response_headers = response_headers_; + updated_response_headers.setDate(not_modified_date); + EXPECT_THAT(not_modified_response_headers, + testing::AllOf(IsSupersetOfHeaders(updated_response_headers), + HeaderHasValueRef(Http::Headers::get().Age, "0"))); + + // The cache is immediate so everything should be done before CacheFilter::encodeData + EXPECT_EQ(filter.encodeData(buffer, true), Http::FilterDataStatus::Continue); + + ::testing::Mock::VerifyAndClearExpectations(&encoder_callbacks_); + + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, DelayedSuccessfulValidation) { + request_headers_.setHost("DelayedSuccessfulValidation"); + const std::string body = "abc"; + + { + // Create filter for request 1 + CacheFilter filter = makeFilter(delayed_cache_); + + // Decode request 1 header + // No hit will be found, so the filter should call continueDecoding + EXPECT_CALL(decoder_callbacks_, continueDecoding); + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // The filter should stop iteration waiting for cache response + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + + // Make the delayed callback to call onHeaders + delayed_cache_.delayed_headers_cb_(); + + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode response + + // Add Etag & Last-Modified headers to the response for validation + response_headers_.setReferenceKey(Http::CustomHeaders::get().Etag, "abc123"); + response_headers_.setReferenceKey(Http::CustomHeaders::get().LastModified, + formatter_.now(time_source_)); + + Buffer::OwnedImpl buffer(body); + response_headers_.setContentLength(body.size()); + EXPECT_EQ(filter.encodeHeaders(response_headers_, false), Http::FilterHeadersStatus::Continue); + EXPECT_EQ(filter.encodeData(buffer, true), Http::FilterDataStatus::Continue); + filter.onDestroy(); + } + { + // Create filter for request 2 + CacheFilter filter = makeFilter(delayed_cache_); + + // Make request require validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + + // Decode request 2 header + // No hit will be found, so the filter should call continueDecoding + EXPECT_CALL(decoder_callbacks_, continueDecoding); + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // The filter should stop iteration waiting for cache response + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + + // Make the delayed callback to call onHeaders + delayed_cache_.delayed_headers_cb_(); + + // Make sure validation conditional headers are added + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-none-match", "abc123"}, {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode 304 response + // Advance time to make sure the cached date is updated with the 304 date + time_source_.advanceTimeWait(std::chrono::seconds(10)); + const std::string not_modified_date = formatter_.now(time_source_); + Http::TestResponseHeaderMapImpl not_modified_response_headers = {{":status", "304"}, + {"date", not_modified_date}}; + + // Check that the cached response body is encoded + Buffer::OwnedImpl buffer(body); + EXPECT_CALL( + encoder_callbacks_, + addEncodedData(testing::Property(&Buffer::Instance::toString, testing::Eq(body)), true)); + + // The cache is delayed so encoding iteration should be stopped until the body is encoded + // StopIteration does not stop encodeData of the same filter from being called + EXPECT_EQ(filter.encodeHeaders(not_modified_response_headers, true), + Http::FilterHeadersStatus::StopIteration); + + // Check for the cached response headers with updated date + Http::TestResponseHeaderMapImpl updated_response_headers = response_headers_; + updated_response_headers.setDate(not_modified_date); + EXPECT_THAT(not_modified_response_headers, + testing::AllOf(IsSupersetOfHeaders(updated_response_headers), + HeaderHasValueRef(Http::Headers::get().Age, "0"))); + + // A 304 response should not have a body, so encodeData should not be called + // However, if a body is present by mistake, encodeData should stop iteration until + // encoding the cached response is done + EXPECT_EQ(filter.encodeData(buffer, true), Http::FilterDataStatus::StopIterationAndBuffer); + + // Delayed call to onBody to encode cached response + delayed_cache_.delayed_body_cb_(); + + ::testing::Mock::VerifyAndClearExpectations(&encoder_callbacks_); + + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, ImmediateUnsuccessfulValidation) { + request_headers_.setHost("ImmediateUnsuccessfulValidation"); + + { + // Create filter for request 1 + CacheFilter filter = makeFilter(simple_cache_); + + // Decode request 1 header + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + + // Encode response + + // Add Etag & Last-Modified headers to the response for validation + response_headers_.setReferenceKey(Http::CustomHeaders::get().Etag, "abc123"); + response_headers_.setReferenceKey(Http::CustomHeaders::get().LastModified, + formatter_.now(time_source_)); + + const std::string body = "abc"; + Buffer::OwnedImpl buffer(body); + response_headers_.setContentLength(body.size()); + EXPECT_EQ(filter.encodeHeaders(response_headers_, false), Http::FilterHeadersStatus::Continue); + EXPECT_EQ(filter.encodeData(buffer, true), Http::FilterDataStatus::Continue); + filter.onDestroy(); + } + { + // Create filter for request 2 + CacheFilter filter = makeFilter(simple_cache_); + + // Make request require validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + + // Decode request 2 header + // Make sure the filter did not encode any headers or data (during decoding) + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::Continue); + + // Make sure validation conditional headers are added + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-none-match", "abc123"}, {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode new response + // Change the status code to make sure new headers are served, not the cached ones + response_headers_.setStatus(201); + + // Check that no cached data is encoded + EXPECT_CALL(encoder_callbacks_, addEncodedData).Times(0); + + // The cache is immediate so everything should be done before encodeHeaders returns + EXPECT_EQ(filter.encodeHeaders(response_headers_, false), Http::FilterHeadersStatus::Continue); + + // Check for the cached response headers with updated date + EXPECT_THAT(response_headers_, HeaderHasValueRef(Http::Headers::get().Status, "201")); + + ::testing::Mock::VerifyAndClearExpectations(&encoder_callbacks_); + + filter.onDestroy(); + } +} + +TEST_F(CacheFilterTest, DelayedUnuccessfulValidation) { + request_headers_.setHost("DelayedUnuccessfulValidation"); + + { + // Create filter for request 1 + CacheFilter filter = makeFilter(delayed_cache_); + + // Decode request 1 header + // No hit will be found, so the filter should call continueDecoding + EXPECT_CALL(decoder_callbacks_, continueDecoding); + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // The filter should stop iteration waiting for cache response + EXPECT_EQ(filter.decodeHeaders(request_headers_, true), + Http::FilterHeadersStatus::StopAllIterationAndWatermark); + + // Make the delayed callback to call onHeaders + delayed_cache_.delayed_headers_cb_(); + + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode response + + // Add Etag & Last-Modified headers to the response for validation + response_headers_.setReferenceKey(Http::CustomHeaders::get().Etag, "abc123"); + response_headers_.setReferenceKey(Http::CustomHeaders::get().LastModified, + formatter_.now(time_source_)); + + const std::string body = "abc"; + Buffer::OwnedImpl buffer(body); + response_headers_.setContentLength(body.size()); + EXPECT_EQ(filter.encodeHeaders(response_headers_, false), Http::FilterHeadersStatus::Continue); + EXPECT_EQ(filter.encodeData(buffer, true), Http::FilterDataStatus::Continue); + filter.onDestroy(); + } + { + // Create filter for request 2 + CacheFilter filter = makeFilter(delayed_cache_); + + // Make request require validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + + // Decode request 2 header + // No hit will be found, so the filter should call continueDecoding + EXPECT_CALL(decoder_callbacks_, continueDecoding); + // Make sure the filter did not encode any headers or data + EXPECT_CALL(decoder_callbacks_, encodeHeaders_).Times(0); + EXPECT_CALL(decoder_callbacks_, encodeData).Times(0); + // The filter should stop iteration waiting for cache response EXPECT_EQ(filter.decodeHeaders(request_headers_, true), Http::FilterHeadersStatus::StopAllIterationAndWatermark); + + // Make the delayed callback to call onHeaders + delayed_cache_.delayed_headers_cb_(); + + // Make sure validation conditional headers are added + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-none-match", "abc123"}, {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + ::testing::Mock::VerifyAndClearExpectations(&decoder_callbacks_); + + // Encode new response + // Change the status code to make sure new headers are served, not the cached ones + response_headers_.setStatus(201); + + // Check that no cached response body is encoded + EXPECT_CALL(encoder_callbacks_, addEncodedData).Times(0); + + // The delayed cache has no effect here as nothing is fetched from cache + EXPECT_EQ(filter.encodeHeaders(response_headers_, false), Http::FilterHeadersStatus::Continue); + + // Check for the cached response headers with updated date + EXPECT_THAT(response_headers_, HeaderHasValueRef(Http::Headers::get().Status, "201")); + + ::testing::Mock::VerifyAndClearExpectations(&encoder_callbacks_); + filter.onDestroy(); } } @@ -200,6 +674,153 @@ TEST_F(CacheFilterTest, GetRequestWithBodyAndTrailers) { } } +// A new type alias for a different type of tests that use the exact same class +using ValidationHeadersTest = CacheFilterTest; + +TEST_F(ValidationHeadersTest, EtagAndLastModified) { + request_headers_.setHost("EtagAndLastModified"); + const std::string etag = "abc123"; + + // Make request 1 to insert the response into cache + { + CacheFilter filter = makeFilter(simple_cache_); + filter.decodeHeaders(request_headers_, true); + + // Add validation headers to the response + response_headers_.setReferenceKey(Http::CustomHeaders::get().Etag, etag); + response_headers_.setReferenceKey(Http::CustomHeaders::get().LastModified, + formatter_.now(time_source_)); + + filter.encodeHeaders(response_headers_, true); + } + // Make request 2 to test for added conditional headers + { + CacheFilter filter = makeFilter(simple_cache_); + + // Make sure the request requires validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + filter.decodeHeaders(request_headers_, true); + + // Make sure validation conditional headers are added + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-none-match", "abc123"}, {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + } +} + +TEST_F(ValidationHeadersTest, EtagOnly) { + request_headers_.setHost("EtagOnly"); + const std::string etag = "abc123"; + + // Make request 1 to insert the response into cache + { + CacheFilter filter = makeFilter(simple_cache_); + filter.decodeHeaders(request_headers_, true); + + // Add validation headers to the response + response_headers_.setReferenceKey(Http::CustomHeaders::get().Etag, etag); + + filter.encodeHeaders(response_headers_, true); + } + // Make request 2 to test for added conditional headers + { + CacheFilter filter = makeFilter(simple_cache_); + + // Make sure the request requires validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + filter.decodeHeaders(request_headers_, true); + + // Make sure validation conditional headers are added + // If-Modified-Since falls back to date + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-none-match", "abc123"}, {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + } +} + +TEST_F(ValidationHeadersTest, LastModifiedOnly) { + request_headers_.setHost("LastModifiedOnly"); + + // Make request 1 to insert the response into cache + { + CacheFilter filter = makeFilter(simple_cache_); + filter.decodeHeaders(request_headers_, true); + + // Add validation headers to the response + response_headers_.setReferenceKey(Http::CustomHeaders::get().LastModified, + formatter_.now(time_source_)); + + filter.encodeHeaders(response_headers_, true); + } + // Make request 2 to test for added conditional headers + { + CacheFilter filter = makeFilter(simple_cache_); + + // Make sure the request requires validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + filter.decodeHeaders(request_headers_, true); + + // Make sure validation conditional headers are added + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + } +} + +TEST_F(ValidationHeadersTest, NoEtagOrLastModified) { + request_headers_.setHost("NoEtagOrLastModified"); + + // Make request 1 to insert the response into cache + { + CacheFilter filter = makeFilter(simple_cache_); + filter.decodeHeaders(request_headers_, true); + filter.encodeHeaders(response_headers_, true); + } + // Make request 2 to test for added conditional headers + { + CacheFilter filter = makeFilter(simple_cache_); + + // Make sure the request requires validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + filter.decodeHeaders(request_headers_, true); + + // Make sure validation conditional headers are added + // If-Modified-Since falls back to date + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + } +} + +TEST_F(ValidationHeadersTest, InvalidLastModified) { + request_headers_.setHost("InvalidLastModified"); + + // Make request 1 to insert the response into cache + { + CacheFilter filter = makeFilter(simple_cache_); + filter.decodeHeaders(request_headers_, true); + + // Add validation headers to the response + response_headers_.setReferenceKey(Http::CustomHeaders::get().LastModified, "invalid-date"); + + filter.encodeHeaders(response_headers_, true); + } + // Make request 2 to test for added conditional headers + { + CacheFilter filter = makeFilter(simple_cache_); + + // Make sure the request requires validation + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + filter.decodeHeaders(request_headers_, true); + + // Make sure validation conditional headers are added + // If-Modified-Since falls back to date + const Http::TestRequestHeaderMapImpl injected_headers = { + {"if-modified-since", formatter_.now(time_source_)}}; + EXPECT_THAT(request_headers_, IsSupersetOfHeaders(injected_headers)); + } +} + } // namespace } // namespace Cache } // namespace HttpFilters diff --git a/test/extensions/filters/http/cache/cache_headers_utils_test.cc b/test/extensions/filters/http/cache/cache_headers_utils_test.cc index dd3f0a78e52b..c9a0f2df7137 100644 --- a/test/extensions/filters/http/cache/cache_headers_utils_test.cc +++ b/test/extensions/filters/http/cache/cache_headers_utils_test.cc @@ -9,6 +9,7 @@ #include "extensions/filters/http/cache/cache_headers_utils.h" +#include "test/extensions/filters/http/cache/common.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -302,13 +303,19 @@ TEST_P(ResponseCacheControlTest, ResponseCacheControlTest) { INSTANTIATE_TEST_SUITE_P(Ok, HttpTimeTest, testing::ValuesIn(HttpTimeTest::getOkTestCases())); -TEST_P(HttpTimeTest, Ok) { +TEST_P(HttpTimeTest, OkFormats) { const Http::TestResponseHeaderMapImpl response_headers{{"date", GetParam()}}; // Manually confirmed that 784111777 is 11/6/94, 8:46:37. EXPECT_EQ(784111777, SystemTime::clock::to_time_t(CacheHeadersUtils::httpTime(response_headers.Date()))); } +TEST(HttpTime, InvalidFormat) { + const std::string invalid_format_date = "Sunday, 06-11-1994 08:49:37"; + const Http::TestResponseHeaderMapImpl response_headers{{"date", invalid_format_date}}; + EXPECT_EQ(CacheHeadersUtils::httpTime(response_headers.Date()), SystemTime()); +} + TEST(HttpTime, Null) { EXPECT_EQ(CacheHeadersUtils::httpTime(nullptr), SystemTime()); } void testReadAndRemoveLeadingDigits(absl::string_view input, int64_t expected, diff --git a/test/extensions/filters/http/cache/cacheability_utils_test.cc b/test/extensions/filters/http/cache/cacheability_utils_test.cc index f4647e8bfc3f..598edf2d2147 100644 --- a/test/extensions/filters/http/cache/cacheability_utils_test.cc +++ b/test/extensions/filters/http/cache/cacheability_utils_test.cc @@ -12,121 +12,137 @@ namespace HttpFilters { namespace Cache { namespace { -class IsCacheableRequestTest : public testing::TestWithParam { +class IsCacheableRequestTest : public testing::Test { protected: - const Http::TestRequestHeaderMapImpl cacheable_request_headers_ = {{":path", "/"}, - {":method", "GET"}, - {"x-forwarded-proto", "http"}, - {":authority", "test.com"}}; + Http::TestRequestHeaderMapImpl request_headers_ = {{":path", "/"}, + {":method", "GET"}, + {"x-forwarded-proto", "http"}, + {":authority", "test.com"}}; +}; + +class RequestConditionalHeadersTest : public testing::TestWithParam { +protected: + Http::TestRequestHeaderMapImpl request_headers_ = {{":path", "/"}, + {":method", "GET"}, + {"x-forwarded-proto", "http"}, + {":authority", "test.com"}}; + std::string conditionalHeader() const { return GetParam(); } }; class IsCacheableResponseTest : public testing::Test { protected: std::string cache_control_ = "max-age=3600"; - const Http::TestResponseHeaderMapImpl cacheable_response_headers_ = { - {":status", "200"}, - {"date", "Sun, 06 Nov 1994 08:49:37 GMT"}, - {"cache-control", cache_control_}}; + Http::TestResponseHeaderMapImpl response_headers_ = {{":status", "200"}, + {"date", "Sun, 06 Nov 1994 08:49:37 GMT"}, + {"cache-control", cache_control_}}; }; TEST_F(IsCacheableRequestTest, CacheableRequest) { - EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(cacheable_request_headers_)); + EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers_)); } TEST_F(IsCacheableRequestTest, PathHeader) { - Http::TestRequestHeaderMapImpl request_headers = cacheable_request_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.removePath(); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.removePath(); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); } TEST_F(IsCacheableRequestTest, HostHeader) { - Http::TestRequestHeaderMapImpl request_headers = cacheable_request_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.removeHost(); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.removeHost(); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); } TEST_F(IsCacheableRequestTest, MethodHeader) { const Http::HeaderValues& header_values = Http::Headers::get(); - Http::TestRequestHeaderMapImpl request_headers = cacheable_request_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.setMethod(header_values.MethodValues.Post); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.setMethod(header_values.MethodValues.Put); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.removeMethod(); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.setMethod(header_values.MethodValues.Post); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.setMethod(header_values.MethodValues.Put); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.removeMethod(); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); } TEST_F(IsCacheableRequestTest, ForwardedProtoHeader) { - Http::TestRequestHeaderMapImpl request_headers = cacheable_request_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.setForwardedProto("ftp"); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.removeForwardedProto(); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.setForwardedProto("ftp"); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.removeForwardedProto(); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); } TEST_F(IsCacheableRequestTest, AuthorizationHeader) { - Http::TestRequestHeaderMapImpl request_headers = cacheable_request_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.setCopy(Http::CustomHeaders::get().Authorization, - "basic YWxhZGRpbjpvcGVuc2VzYW1l"); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.setReferenceKey(Http::CustomHeaders::get().Authorization, + "basic YWxhZGRpbjpvcGVuc2VzYW1l"); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); } -INSTANTIATE_TEST_SUITE_P(ConditionalHeaders, IsCacheableRequestTest, +INSTANTIATE_TEST_SUITE_P(ConditionalHeaders, RequestConditionalHeadersTest, testing::Values("if-match", "if-none-match", "if-modified-since", - "if-unmodified-since", "if-range")); - -TEST_P(IsCacheableRequestTest, ConditionalHeaders) { - Http::TestRequestHeaderMapImpl request_headers = cacheable_request_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers)); - request_headers.setCopy(Http::LowerCaseString{GetParam()}, "test-value"); - EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers)); + "if-unmodified-since", "if-range"), + [](const auto& info) { + std::string test_name = info.param; + absl::c_replace_if( + test_name, [](char c) { return !std::isalnum(c); }, '_'); + return test_name; + }); + +TEST_P(RequestConditionalHeadersTest, ConditionalHeaders) { + EXPECT_TRUE(CacheabilityUtils::isCacheableRequest(request_headers_)); + request_headers_.setCopy(Http::LowerCaseString{conditionalHeader()}, "test-value"); + EXPECT_FALSE(CacheabilityUtils::isCacheableRequest(request_headers_)); } TEST_F(IsCacheableResponseTest, CacheableResponse) { - EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(cacheable_response_headers_)); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); } TEST_F(IsCacheableResponseTest, UncacheableStatusCode) { - Http::TestResponseHeaderMapImpl response_headers = cacheable_response_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers)); - response_headers.setStatus("700"); - EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers)); - response_headers.removeStatus(); - EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); + response_headers_.setStatus("700"); + EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers_)); + response_headers_.removeStatus(); + EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers_)); } TEST_F(IsCacheableResponseTest, ValidationData) { - Http::TestResponseHeaderMapImpl response_headers = cacheable_response_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers)); - response_headers.setCopy(Http::CustomHeaders::get().CacheControl, "s-maxage=1000"); - EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers)); - response_headers.setCopy(Http::CustomHeaders::get().CacheControl, "public, no-transform"); - EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers)); - response_headers.remove(Http::CustomHeaders::get().CacheControl); - EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers)); - response_headers.setCopy(Http::Headers::get().Expires, "Sun, 06 Nov 1994 09:49:37 GMT"); - EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); + // No cache control headers or expires header + response_headers_.remove(Http::CustomHeaders::get().CacheControl); + EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers_)); + // No max-age data or expires header + response_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, + "public, no-transform"); + EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers_)); + // Max-age data available + response_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "s-maxage=1000"); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); + // Max-age data available with no date + response_headers_.removeDate(); + EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers_)); + // No date, but the response requires revalidation anyway + response_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "no-cache"); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); + // No cache control headers or date, but there is an expires header + response_headers_.setReferenceKey(Http::Headers::get().Expires, "Sun, 06 Nov 1994 09:49:37 GMT"); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); } TEST_F(IsCacheableResponseTest, ResponseNoStore) { - Http::TestResponseHeaderMapImpl response_headers = cacheable_response_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); std::string cache_control_no_store = absl::StrCat(cache_control_, ", no-store"); - response_headers.setCopy(Http::CustomHeaders::get().CacheControl, cache_control_no_store); - EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers)); + response_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, + cache_control_no_store); + EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers_)); } TEST_F(IsCacheableResponseTest, ResponsePrivate) { - Http::TestResponseHeaderMapImpl response_headers = cacheable_response_headers_; - EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers)); + EXPECT_TRUE(CacheabilityUtils::isCacheableResponse(response_headers_)); std::string cache_control_private = absl::StrCat(cache_control_, ", private"); - response_headers.setCopy(Http::CustomHeaders::get().CacheControl, cache_control_private); - EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers)); + response_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, cache_control_private); + EXPECT_FALSE(CacheabilityUtils::isCacheableResponse(response_headers_)); } } // namespace diff --git a/test/extensions/filters/http/cache/common.h b/test/extensions/filters/http/cache/common.h new file mode 100644 index 000000000000..6a94861bd8db --- /dev/null +++ b/test/extensions/filters/http/cache/common.h @@ -0,0 +1,134 @@ +#pragma once + +#include "extensions/filters/http/cache/cache_headers_utils.h" +#include "extensions/filters/http/cache/http_cache.h" +#include "extensions/filters/http/cache/simple_http_cache/simple_http_cache.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Cache { + +// Wrapper for SimpleHttpCache that delays the onHeaders/onBody/onTrailers callbacks from +// getHeaders/getBody/getTrailers; for verifying that CacheFilter works correctly whether the +// callbacks happen immediately, or after getHeaders/getBody/getTrailers return +// Used to test the synchronization made using get_headers_state_ & encode_cached_response_state_ +class DelayedCache : public SimpleHttpCache { +public: + // HttpCache + LookupContextPtr makeLookupContext(LookupRequest&& request) override { + return std::make_unique( + SimpleHttpCache::makeLookupContext(std::move(request)), + DelayedCallbacks{delayed_headers_cb_, delayed_body_cb_, delayed_trailers_cb_}); + } + InsertContextPtr makeInsertContext(LookupContextPtr&& lookup_context) override { + return SimpleHttpCache::makeInsertContext( + std::move(dynamic_cast(*lookup_context).context_)); + } + + std::function delayed_headers_cb_, delayed_body_cb_, delayed_trailers_cb_; + +private: + struct DelayedCallbacks { + std::function&headers_cb_, &body_cb_, &trailers_cb_; + }; + class DelayedLookupContext : public LookupContext { + public: + DelayedLookupContext(LookupContextPtr&& context, DelayedCallbacks delayed_callbacks) + : context_(std::move(context)), delayed_callbacks_(delayed_callbacks) {} + void getHeaders(LookupHeadersCallback&& cb) override { + delayed_callbacks_.headers_cb_ = [this, cb]() mutable { + context_->getHeaders(std::move(cb)); + }; + } + void getBody(const AdjustedByteRange& range, LookupBodyCallback&& cb) override { + delayed_callbacks_.body_cb_ = [this, cb, range]() mutable { + context_->getBody(range, std::move(cb)); + }; + } + void getTrailers(LookupTrailersCallback&& cb) override { + delayed_callbacks_.trailers_cb_ = [this, cb]() mutable { + context_->getTrailers(std::move(cb)); + }; + } + + LookupContextPtr context_; + DelayedCallbacks delayed_callbacks_; + }; +}; + +std::ostream& operator<<(std::ostream& os, const RequestCacheControl& request_cache_control) { + std::string s = "{"; + s += request_cache_control.must_validate_ ? "must_validate, " : ""; + s += request_cache_control.no_store_ ? "no_store, " : ""; + s += request_cache_control.no_transform_ ? "no_transform, " : ""; + s += request_cache_control.only_if_cached_ ? "only_if_cached, " : ""; + + s += request_cache_control.max_age_.has_value() + ? "max-age=" + std::to_string(request_cache_control.max_age_.value().count()) + ", " + : ""; + s += request_cache_control.min_fresh_.has_value() + ? "min-fresh=" + std::to_string(request_cache_control.min_fresh_.value().count()) + ", " + : ""; + s += request_cache_control.max_stale_.has_value() + ? "max-stale=" + std::to_string(request_cache_control.max_stale_.value().count()) + ", " + : ""; + + // Remove any extra ", " at the end + if (s.size() > 1) { + s.pop_back(); + s.pop_back(); + } + + s += "}"; + return os << s; +} + +std::ostream& operator<<(std::ostream& os, const ResponseCacheControl& response_cache_control) { + std::string s = "{"; + s += response_cache_control.must_validate_ ? "must_validate, " : ""; + s += response_cache_control.no_store_ ? "no_store, " : ""; + s += response_cache_control.no_transform_ ? "no_transform, " : ""; + s += response_cache_control.no_stale_ ? "no_stale, " : ""; + s += response_cache_control.is_public_ ? "public, " : ""; + + s += response_cache_control.max_age_.has_value() + ? "max-age=" + std::to_string(response_cache_control.max_age_.value().count()) + ", " + : ""; + + // Remove any extra ", " at the end + if (s.size() > 1) { + s.pop_back(); + s.pop_back(); + } + + s += "}"; + return os << s; +} + +std::ostream& operator<<(std::ostream& os, CacheEntryStatus status) { + switch (status) { + case CacheEntryStatus::Ok: + return os << "Ok"; + case CacheEntryStatus::Unusable: + return os << "Unusable"; + case CacheEntryStatus::RequiresValidation: + return os << "RequiresValidation"; + case CacheEntryStatus::FoundNotModified: + return os << "FoundNotModified"; + case CacheEntryStatus::SatisfiableRange: + return os << "SatisfiableRange"; + case CacheEntryStatus::NotSatisfiableRange: + return os << "NotSatisfiableRange"; + } + NOT_REACHED_GCOVR_EXCL_LINE; +} + +std::ostream& operator<<(std::ostream& os, const AdjustedByteRange& range) { + return os << "[" << range.begin() << "," << range.end() << ")"; +} + +} // namespace Cache +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/http/cache/config_test.cc b/test/extensions/filters/http/cache/config_test.cc index 2b05de007256..83459d8328ef 100644 --- a/test/extensions/filters/http/cache/config_test.cc +++ b/test/extensions/filters/http/cache/config_test.cc @@ -42,6 +42,7 @@ TEST_F(CacheFilterFactoryTest, UnregisteredTypedConfig) { envoy::extensions::filters::http::cache::v3alpha::CacheConfig()); EXPECT_THROW(factory_.createFilterFactoryFromProto(config_, "stats", context_), EnvoyException); } + } // namespace } // namespace Cache } // namespace HttpFilters diff --git a/test/extensions/filters/http/cache/http_cache_test.cc b/test/extensions/filters/http/cache/http_cache_test.cc index fac7c099f73d..f4e0361156e6 100644 --- a/test/extensions/filters/http/cache/http_cache_test.cc +++ b/test/extensions/filters/http/cache/http_cache_test.cc @@ -1,6 +1,9 @@ +#include + #include "extensions/filters/http/cache/cache_headers_utils.h" #include "extensions/filters/http/cache/http_cache.h" +#include "test/extensions/filters/http/cache/common.h" #include "test/mocks/http/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -15,60 +18,135 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Cache { -TEST(RawByteRangeTest, IsSuffix) { - auto r = RawByteRange(UINT64_MAX, 4); - ASSERT_TRUE(r.isSuffix()); -} - -TEST(RawByteRangeTest, IsNotSuffix) { - auto r = RawByteRange(3, 4); - ASSERT_FALSE(r.isSuffix()); -} - -TEST(RawByteRangeTest, FirstBytePos) { - auto r = RawByteRange(3, 4); - ASSERT_EQ(3, r.firstBytePos()); -} - -TEST(RawByteRangeTest, LastBytePos) { - auto r = RawByteRange(3, 4); - ASSERT_EQ(4, r.lastBytePos()); -} - -TEST(RawByteRangeTest, SuffixLength) { - auto r = RawByteRange(UINT64_MAX, 4); - ASSERT_EQ(4, r.suffixLength()); -} - -TEST(AdjustedByteRangeTest, Length) { - auto a = AdjustedByteRange(3, 6); - ASSERT_EQ(3, a.length()); -} - -TEST(AdjustedByteRangeTest, TrimFront) { - auto a = AdjustedByteRange(3, 6); - a.trimFront(2); - ASSERT_EQ(5, a.begin()); -} +namespace { -TEST(AdjustedByteRangeTest, MaxLength) { - auto a = AdjustedByteRange(0, UINT64_MAX); - ASSERT_EQ(UINT64_MAX, a.length()); -} +struct LookupRequestTestCase { + std::string test_name, request_cache_control, response_cache_control; + SystemTime request_time, response_time; + CacheEntryStatus expected_cache_entry_status; +}; -TEST(AdjustedByteRangeTest, MaxTrim) { - auto a = AdjustedByteRange(0, UINT64_MAX); - a.trimFront(UINT64_MAX); - ASSERT_EQ(0, a.length()); -} +using Seconds = std::chrono::seconds; -class LookupRequestTest : public testing::Test { -protected: - Event::SimulatedTimeSystem time_source_; - SystemTime current_time_ = time_source_.systemTime(); +class LookupRequestTest : public testing::TestWithParam { +public: DateFormatter formatter_{"%a, %d %b %Y %H:%M:%S GMT"}; - Http::TestRequestHeaderMapImpl request_headers_{ - {":path", "/"}, {"x-forwarded-proto", "https"}, {":authority", "example.com"}}; + Http::TestRequestHeaderMapImpl request_headers_{{":path", "/"}, + {":method", "GET"}, + {"x-forwarded-proto", "https"}, + {":authority", "example.com"}}; + + static const SystemTime& currentTime() { + CONSTRUCT_ON_FIRST_USE(SystemTime, Event::SimulatedTimeSystem().systemTime()); + } + + static const std::vector& getTestCases() { + CONSTRUCT_ON_FIRST_USE(std::vector, + {"request_requires_revalidation", + /*request_cache_control=*/"no-cache", + /*response_cache_control=*/"public, max-age=3600", + /*request_time=*/currentTime(), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"response_requires_revalidation", + /*request_cache_control=*/"", + /*response_cache_control=*/"no-cache", + /*request_time=*/currentTime(), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"future_response_time", + /*request_cache_control=*/"", + /*response_cache_control=*/"public, max-age=3600", + /*request_time=*/currentTime(), + /*response_time=*/currentTime() + Seconds(1), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"request_max_age_satisfied", + /*request_cache_control=*/"max-age=10", + /*response_cache_control=*/"public, max-age=3600", + /*request_time=*/currentTime() + Seconds(9), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::Ok}, + {"request_max_age_unsatisfied", + /*request_cache_control=*/"max-age=10", + /*response_cache_control=*/"public, max-age=3600", + /*request_time=*/currentTime() + Seconds(11), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"request_min_fresh_satisfied", + /*request_cache_control=*/"min-fresh=1000", + /*response_cache_control=*/"public, max-age=2000", + /*request_time=*/currentTime() + Seconds(999), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::Ok}, + {"request_min_fresh_unsatisfied", + /*request_cache_control=*/"min-fresh=1000", + /*response_cache_control=*/"public, max-age=2000", + /*request_time=*/currentTime() + Seconds(1001), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"request_max_age_satisfied_but_min_fresh_unsatisfied", + /*request_cache_control=*/"max-age=1500, min-fresh=1000", + /*response_cache_control=*/"public, max-age=2000", + /*request_time=*/currentTime() + Seconds(1001), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"request_max_age_satisfied_but_max_stale_unsatisfied", + /*request_cache_control=*/"max-age=1500, max-stale=400", + /*response_cache_control=*/"public, max-age=1000", + /*request_time=*/currentTime() + Seconds(1401), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"request_max_stale_satisfied_but_min_fresh_unsatisfied", + /*request_cache_control=*/"min-fresh=1000, max-stale=500", + /*response_cache_control=*/"public, max-age=2000", + /*request_time=*/currentTime() + Seconds(1001), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"request_max_stale_satisfied_but_max_age_unsatisfied", + /*request_cache_control=*/"max-age=1200, max-stale=500", + /*response_cache_control=*/"public, max-age=1000", + /*request_time=*/currentTime() + Seconds(1201), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"request_min_fresh_satisfied_but_max_age_unsatisfied", + /*request_cache_control=*/"max-age=500, min-fresh=400", + /*response_cache_control=*/"public, max-age=1000", + /*request_time=*/currentTime() + Seconds(501), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"expired", + /*request_cache_control=*/"", + /*response_cache_control=*/"public, max-age=1000", + /*request_time=*/currentTime() + Seconds(1001), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"expired_but_max_stale_satisfied", + /*request_cache_control=*/"max-stale=500", + /*response_cache_control=*/"public, max-age=1000", + /*request_time=*/currentTime() + Seconds(1499), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::Ok}, + {"expired_max_stale_unsatisfied", + /*request_cache_control=*/"max-stale=500", + /*response_cache_control=*/"public, max-age=1000", + /*request_time=*/currentTime() + Seconds(1501), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"expired_max_stale_satisfied_but_response_must_revalidate", + /*request_cache_control=*/"max-stale=500", + /*response_cache_control=*/"public, max-age=1000, must-revalidate", + /*request_time=*/currentTime() + Seconds(1499), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::RequiresValidation}, + {"fresh_and_response_must_revalidate", + /*request_cache_control=*/"", + /*response_cache_control=*/"public, max-age=1000, must-revalidate", + /*request_time=*/currentTime() + Seconds(999), + /*response_time=*/currentTime(), + /*expected_result=*/CacheEntryStatus::Ok}, + + ); + } }; LookupResult makeLookupResult(const LookupRequest& lookup_request, @@ -78,12 +156,21 @@ LookupResult makeLookupResult(const LookupRequest& lookup_request, std::make_unique(response_headers), content_length); } -TEST_F(LookupRequestTest, MakeLookupResultNoBody) { - const LookupRequest lookup_request(request_headers_, current_time_); +INSTANTIATE_TEST_SUITE_P(ResultMatchesExpectation, LookupRequestTest, + testing::ValuesIn(LookupRequestTest::getTestCases()), + [](const auto& info) { return info.param.test_name; }); + +TEST_P(LookupRequestTest, ResultWithoutBodyMatchesExpectation) { + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, + GetParam().request_cache_control); + const SystemTime request_time = GetParam().request_time, response_time = GetParam().response_time; + const LookupRequest lookup_request(request_headers_, request_time); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, {"cache-control", "public, max-age=3600"}}); + {{"cache-control", GetParam().response_cache_control}, + {"date", formatter_.fromTime(response_time)}}); const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); - ASSERT_EQ(CacheEntryStatus::Ok, lookup_response.cache_entry_status_); + + EXPECT_EQ(GetParam().expected_cache_entry_status, lookup_response.cache_entry_status_); ASSERT_TRUE(lookup_response.headers_); EXPECT_THAT(*lookup_response.headers_, Http::IsSupersetOfHeaders(response_headers)); EXPECT_EQ(lookup_response.content_length_, 0); @@ -91,69 +178,41 @@ TEST_F(LookupRequestTest, MakeLookupResultNoBody) { EXPECT_FALSE(lookup_response.has_trailers_); } -TEST_F(LookupRequestTest, MakeLookupResultBody) { - const LookupRequest lookup_request(request_headers_, current_time_); +TEST_P(LookupRequestTest, ResultWithBodyMatchesExpectation) { + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, + GetParam().request_cache_control); + const SystemTime request_time = GetParam().request_time, response_time = GetParam().response_time; + const LookupRequest lookup_request(request_headers_, request_time); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, {"cache-control", "public, max-age=3600"}}); + {{"cache-control", GetParam().response_cache_control}, + {"date", formatter_.fromTime(response_time)}}); const uint64_t content_length = 5; const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers, content_length); - ASSERT_EQ(CacheEntryStatus::Ok, lookup_response.cache_entry_status_); - ASSERT_TRUE(lookup_response.headers_); - EXPECT_THAT(*lookup_response.headers_, Http::IsSupersetOfHeaders(response_headers)); - EXPECT_EQ(lookup_response.content_length_, content_length); - EXPECT_TRUE(lookup_response.response_ranges_.empty()); - EXPECT_FALSE(lookup_response.has_trailers_); -} - -TEST_F(LookupRequestTest, PrivateResponse) { - const LookupRequest lookup_request(request_headers_, current_time_); - const Http::TestResponseHeaderMapImpl response_headers( - {{"age", "2"}, - {"cache-control", "private, max-age=3600"}, - {"date", formatter_.fromTime(current_time_)}}); - const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); - - // We must make sure at cache insertion time, private responses must not be - // inserted. However, if the insertion did happen, it would be served at the - // time of lookup. (Nothing should rely on this.) - ASSERT_EQ(CacheEntryStatus::Ok, lookup_response.cache_entry_status_); - ASSERT_TRUE(lookup_response.headers_); - EXPECT_THAT(*lookup_response.headers_, Http::IsSupersetOfHeaders(response_headers)); - EXPECT_EQ(lookup_response.content_length_, 0); - EXPECT_TRUE(lookup_response.response_ranges_.empty()); - EXPECT_FALSE(lookup_response.has_trailers_); -} -TEST_F(LookupRequestTest, Expired) { - const LookupRequest lookup_request(request_headers_, current_time_); - const Http::TestResponseHeaderMapImpl response_headers( - {{"cache-control", "public, max-age=3600"}, {"date", "Thu, 01 Jan 2019 00:00:00 GMT"}}); - const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); - - EXPECT_EQ(CacheEntryStatus::RequiresValidation, lookup_response.cache_entry_status_); + EXPECT_EQ(GetParam().expected_cache_entry_status, lookup_response.cache_entry_status_); ASSERT_TRUE(lookup_response.headers_); EXPECT_THAT(*lookup_response.headers_, Http::IsSupersetOfHeaders(response_headers)); - EXPECT_EQ(lookup_response.content_length_, 0); + EXPECT_EQ(lookup_response.content_length_, content_length); EXPECT_TRUE(lookup_response.response_ranges_.empty()); EXPECT_FALSE(lookup_response.has_trailers_); } TEST_F(LookupRequestTest, ExpiredViaFallbackheader) { - const LookupRequest lookup_request(request_headers_, current_time_); + const LookupRequest lookup_request(request_headers_, currentTime()); const Http::TestResponseHeaderMapImpl response_headers( - {{"expires", formatter_.fromTime(current_time_ - std::chrono::seconds(5))}, - {"date", formatter_.fromTime(current_time_)}}); + {{"expires", formatter_.fromTime(currentTime() - Seconds(5))}, + {"date", formatter_.fromTime(currentTime())}}); const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); EXPECT_EQ(CacheEntryStatus::RequiresValidation, lookup_response.cache_entry_status_); } TEST_F(LookupRequestTest, NotExpiredViaFallbackheader) { - const LookupRequest lookup_request(request_headers_, current_time_); + const LookupRequest lookup_request(request_headers_, currentTime()); const Http::TestResponseHeaderMapImpl response_headers( - {{"expires", formatter_.fromTime(current_time_ + std::chrono::seconds(5))}, - {"date", formatter_.fromTime(current_time_)}}); + {{"expires", formatter_.fromTime(currentTime() + Seconds(5))}, + {"date", formatter_.fromTime(currentTime())}}); const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); EXPECT_EQ(CacheEntryStatus::Ok, lookup_response.cache_entry_status_); } @@ -162,54 +221,54 @@ TEST_F(LookupRequestTest, NotExpiredViaFallbackheader) { // "Pragma:no-cache" is equivalent to "Cache-Control:no-cache". // https://httpwg.org/specs/rfc7234.html#header.pragma TEST_F(LookupRequestTest, PragmaNoCacheFallback) { - request_headers_.addCopy("pragma", "no-cache"); - const LookupRequest lookup_request(request_headers_, current_time_); + request_headers_.setReferenceKey(Http::CustomHeaders::get().Pragma, "no-cache"); + const LookupRequest lookup_request(request_headers_, currentTime()); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, {"cache-control", "public, max-age=3600"}}); + {{"date", formatter_.fromTime(currentTime())}, {"cache-control", "public, max-age=3600"}}); const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); // Response is not expired but the request requires revalidation through Pragma: no-cache. EXPECT_EQ(CacheEntryStatus::RequiresValidation, lookup_response.cache_entry_status_); } TEST_F(LookupRequestTest, PragmaNoCacheFallbackExtraDirectivesIgnored) { - request_headers_.addCopy("pragma", "no-cache, custom-directive=custom-value"); - const LookupRequest lookup_request(request_headers_, current_time_); + request_headers_.setReferenceKey(Http::CustomHeaders::get().Pragma, + "no-cache, custom-directive=custom-value"); + const LookupRequest lookup_request(request_headers_, currentTime()); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, {"cache-control", "public, max-age=3600"}}); + {{"date", formatter_.fromTime(currentTime())}, {"cache-control", "public, max-age=3600"}}); const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); // Response is not expired but the request requires revalidation through Pragma: no-cache. EXPECT_EQ(CacheEntryStatus::RequiresValidation, lookup_response.cache_entry_status_); } TEST_F(LookupRequestTest, PragmaFallbackOtherValuesIgnored) { - request_headers_.addCopy("pragma", "max-age=0"); - const LookupRequest lookup_request(request_headers_, current_time_ + std::chrono::seconds(5)); + request_headers_.setReferenceKey(Http::CustomHeaders::get().Pragma, "max-age=0"); + const LookupRequest lookup_request(request_headers_, currentTime() + std::chrono::seconds(5)); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, {"cache-control", "public, max-age=3600"}}); + {{"date", formatter_.fromTime(currentTime())}, {"cache-control", "public, max-age=3600"}}); const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); // Response is fresh, Pragma header with values other than "no-cache" is ignored. EXPECT_EQ(CacheEntryStatus::Ok, lookup_response.cache_entry_status_); } TEST_F(LookupRequestTest, PragmaNoFallback) { - request_headers_.addCopy("pragma", "no-cache"); - request_headers_.addCopy("cache-control", "max-age=10"); - const LookupRequest lookup_request(request_headers_, current_time_ + std::chrono::seconds(5)); + request_headers_.setReferenceKey(Http::CustomHeaders::get().Pragma, "no-cache"); + request_headers_.setReferenceKey(Http::CustomHeaders::get().CacheControl, "max-age=10"); + const LookupRequest lookup_request(request_headers_, currentTime() + std::chrono::seconds(5)); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, {"cache-control", "public, max-age=3600"}}); + {{"date", formatter_.fromTime(currentTime())}, {"cache-control", "public, max-age=3600"}}); const LookupResult lookup_response = makeLookupResult(lookup_request, response_headers); // Pragma header is ignored when Cache-Control header is present. EXPECT_EQ(CacheEntryStatus::Ok, lookup_response.cache_entry_status_); } TEST_F(LookupRequestTest, SatisfiableRange) { - // add method (GET) and range to headers - request_headers_.addReference(Http::Headers::get().Method, Http::Headers::get().MethodValues.Get); + // add range to headers request_headers_.addReference(Http::Headers::get().Range, "bytes=1-99,3-,-2"); - const LookupRequest lookup_request(request_headers_, current_time_); + const LookupRequest lookup_request(request_headers_, currentTime()); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, + {{"date", formatter_.fromTime(currentTime())}, {"cache-control", "public, max-age=3600"}, {"content-length", "4"}}); const uint64_t content_length = 4; @@ -240,14 +299,13 @@ TEST_F(LookupRequestTest, SatisfiableRange) { } TEST_F(LookupRequestTest, NotSatisfiableRange) { - // add method (GET) and range headers - request_headers_.addReference(Http::Headers::get().Method, Http::Headers::get().MethodValues.Get); + // add range headers request_headers_.addReference(Http::Headers::get().Range, "bytes=5-99,100-"); - const LookupRequest lookup_request(request_headers_, current_time_); + const LookupRequest lookup_request(request_headers_, currentTime()); const Http::TestResponseHeaderMapImpl response_headers( - {{"date", formatter_.fromTime(current_time_)}, + {{"date", formatter_.fromTime(currentTime())}, {"cache-control", "public, max-age=3600"}, {"content-length", "4"}}); const uint64_t content_length = 4; @@ -262,6 +320,53 @@ TEST_F(LookupRequestTest, NotSatisfiableRange) { EXPECT_FALSE(lookup_response.has_trailers_); } +TEST(RawByteRangeTest, IsSuffix) { + auto r = RawByteRange(UINT64_MAX, 4); + ASSERT_TRUE(r.isSuffix()); +} + +TEST(RawByteRangeTest, IsNotSuffix) { + auto r = RawByteRange(3, 4); + ASSERT_FALSE(r.isSuffix()); +} + +TEST(RawByteRangeTest, FirstBytePos) { + auto r = RawByteRange(3, 4); + ASSERT_EQ(3, r.firstBytePos()); +} + +TEST(RawByteRangeTest, LastBytePos) { + auto r = RawByteRange(3, 4); + ASSERT_EQ(4, r.lastBytePos()); +} + +TEST(RawByteRangeTest, SuffixLength) { + auto r = RawByteRange(UINT64_MAX, 4); + ASSERT_EQ(4, r.suffixLength()); +} + +TEST(AdjustedByteRangeTest, Length) { + auto a = AdjustedByteRange(3, 6); + ASSERT_EQ(3, a.length()); +} + +TEST(AdjustedByteRangeTest, TrimFront) { + auto a = AdjustedByteRange(3, 6); + a.trimFront(2); + ASSERT_EQ(5, a.begin()); +} + +TEST(AdjustedByteRangeTest, MaxLength) { + auto a = AdjustedByteRange(0, UINT64_MAX); + ASSERT_EQ(UINT64_MAX, a.length()); +} + +TEST(AdjustedByteRangeTest, MaxTrim) { + auto a = AdjustedByteRange(0, UINT64_MAX); + a.trimFront(UINT64_MAX); + ASSERT_EQ(0, a.length()); +} + struct AdjustByteRangeParams { std::vector request; std::vector result; @@ -469,6 +574,7 @@ TEST_P(ParseInvalidRangeHeaderTest, InvalidRangeReturnsEmpty) { ASSERT_EQ(0, result_vector.size()); } +} // namespace } // namespace Cache } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/cache/simple_http_cache/BUILD b/test/extensions/filters/http/cache/simple_http_cache/BUILD index 3030d84eeae9..c5f39562f6e2 100644 --- a/test/extensions/filters/http/cache/simple_http_cache/BUILD +++ b/test/extensions/filters/http/cache/simple_http_cache/BUILD @@ -14,6 +14,7 @@ envoy_extension_cc_test( extension_name = "envoy.filters.http.cache.simple_http_cache", deps = [ "//source/extensions/filters/http/cache/simple_http_cache:simple_http_cache_lib", + "//test/extensions/filters/http/cache:common", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", ], diff --git a/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc b/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc index 301009223163..acafa5037f55 100644 --- a/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc +++ b/test/extensions/filters/http/cache/simple_http_cache/simple_http_cache_test.cc @@ -6,6 +6,7 @@ #include "extensions/filters/http/cache/cache_headers_utils.h" #include "extensions/filters/http/cache/simple_http_cache/simple_http_cache.h" +#include "test/extensions/filters/http/cache/common.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 4ffedaa0d467..1d74fa10a296 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -972,6 +972,7 @@ restarter resync retriable retriggers +revalidated revalidation rmdir rocketmq From 6e91a56853375d73b0da0d2dcdc2fc31cdb8f4aa Mon Sep 17 00:00:00 2001 From: Sotiris Nanopoulos Date: Wed, 12 Aug 2020 16:05:46 -0700 Subject: [PATCH 49/67] Reverts proxy protocol test on windows (#12619) Signed-off-by: Sotiris Nanopoulos --- test/extensions/filters/listener/proxy_protocol/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/test/extensions/filters/listener/proxy_protocol/BUILD b/test/extensions/filters/listener/proxy_protocol/BUILD index fa746dbaa5c6..f37389795778 100644 --- a/test/extensions/filters/listener/proxy_protocol/BUILD +++ b/test/extensions/filters/listener/proxy_protocol/BUILD @@ -15,6 +15,7 @@ envoy_extension_cc_test( name = "proxy_protocol_test", srcs = ["proxy_protocol_test.cc"], extension_name = "envoy.filters.listener.proxy_protocol", + tags = ["fails_on_windows"], deps = [ "//source/common/buffer:buffer_lib", "//source/common/event:dispatcher_includes", From 2c0b52b23820f7152126ad6181aa07f5f463fcbd Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Thu, 13 Aug 2020 08:12:48 -0400 Subject: [PATCH 50/67] Added a missing extension point to documentation. (#12620) Signed-off-by: Kevin Baichoo Commit Message: Updated the extending envoy documentation to include a missing extension point. Additional Description: See PR #12416 for additional context of this. Risk Level: low Testing: built the documents to ensure they generate correctly. Docs Changes: Release Notes: --- docs/root/extending/extending.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/extending/extending.rst b/docs/root/extending/extending.rst index 1e63c9e99b94..316b43547835 100644 --- a/docs/root/extending/extending.rst +++ b/docs/root/extending/extending.rst @@ -21,6 +21,7 @@ types including: * :ref:`Request ID ` * Transport sockets * BoringSSL private key methods +* :ref:`Internal redirect policy ` As of this writing there is no high level extension developer documentation. The :repo:`existing extensions ` are a good way to learn what is possible. From 4215921ff621fc0142bcce739d98559c59c3a5b8 Mon Sep 17 00:00:00 2001 From: DongRyeol Cha Date: Thu, 13 Aug 2020 21:19:55 +0900 Subject: [PATCH 51/67] Fix regression of /build_* in gitignore (#12630) Commit Message: This is a regression of 887637c that should be retained. This patch re-add it into the gitignore file. Additional Description: None Risk Level: Low Testing: git status Docs Changes: None Release Notes: None Signed-off-by: DongRyeol Cha --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8884ac50e344..a030c858c372 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /bazel-* BROWSE /build +/build_* *.bzlc .cache .clangd From 9e019c3c6c14441382e4053598df504252ec17ce Mon Sep 17 00:00:00 2001 From: Jose Ulises Nino Rivera Date: Thu, 13 Aug 2020 08:38:42 -0700 Subject: [PATCH 52/67] statsd: revert visibility to public (#12621) Commit Message: reverts visibility of the statd sink to public. Risk Level: low Testing: local build, CI Signed-off-by: Jose Nino --- source/extensions/stat_sinks/statsd/BUILD | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/extensions/stat_sinks/statsd/BUILD b/source/extensions/stat_sinks/statsd/BUILD index 0a8ed4648bca..82ee8f026cc0 100644 --- a/source/extensions/stat_sinks/statsd/BUILD +++ b/source/extensions/stat_sinks/statsd/BUILD @@ -16,10 +16,6 @@ envoy_cc_extension( hdrs = ["config.h"], security_posture = "data_plane_agnostic", # Legacy test use. TODO(#9953) clean up. - visibility = [ - "//:extension_config", - "//test/server:__subpackages__", - ], deps = [ "//include/envoy/registry", "//source/common/network:address_lib", From 9d03c574a0e4308589fbd1ac5731bfb455b51d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Thu, 13 Aug 2020 12:04:23 -0400 Subject: [PATCH 53/67] router: add dynamic metadata header formatter (#11858) This new header formatter acts in the opposite direction of the header-to-metadata filter. That is, it allows setting a header from what's available in a request's metadata. We need this to populate some request headers based on what was previously extracted and transformed via the header-to-metadata filter. Risk Level: low Testing: unit test Docs Changes: yes Release Notes: added Signed-off-by: Raul Gutierrez Segales --- .../http/http_conn_man/headers.rst | 8 ++++++- docs/root/version_history/current.rst | 1 + source/common/router/header_formatter.cc | 22 +++++++++++++------ test/common/router/header_formatter_test.cc | 15 +++++++++++++ 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/root/configuration/http/http_conn_man/headers.rst b/docs/root/configuration/http/http_conn_man/headers.rst index 0ef3e630c5e0..e95b6479d8bb 100644 --- a/docs/root/configuration/http/http_conn_man/headers.rst +++ b/docs/root/configuration/http/http_conn_man/headers.rst @@ -621,6 +621,12 @@ Supported variable names are: Upstream metadata cannot be added to request headers as the upstream host has not been selected when custom request headers are generated. +%DYNAMIC_METADATA(["namespace", "key", ...])% + Similar to UPSTREAM_METADATA, populates the header with dynamic metadata available in a request + (e.g.: added by filters like the header-to-metadata filter). + + This works both on request and response headers. + %UPSTREAM_REMOTE_ADDRESS% Remote address of the upstream host. If the address is an IP address it includes both address and port. The upstream remote address cannot be added to request headers as the upstream host @@ -657,4 +663,4 @@ Supported variable names are: %RESPONSE_CODE_DETAILS% Response code details provides additional information about the HTTP response code, such as - who set it (the upstream or envoy) and why. \ No newline at end of file + who set it (the upstream or envoy) and why. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 9ca7c87a0c55..a8a8b0324ff8 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -68,6 +68,7 @@ New Features retry policy, which allows retrying envoy's own rate limited responses. * router: added new :ref:`host_rewrite_path_regex ` option, which allows rewriting Host header based on path. +* router: added support for DYNAMIC_METADATA :ref:`header formatter `. * signal: added support for calling fatal error handlers without envoy's signal handler, via FatalErrorHandler::callFatalErrorHandlers(). * stats: added optional histograms to :ref:`cluster stats ` that track headers and body sizes of requests and responses. diff --git a/source/common/router/header_formatter.cc b/source/common/router/header_formatter.cc index 8e40e95b23eb..b8ff26677261 100644 --- a/source/common/router/header_formatter.cc +++ b/source/common/router/header_formatter.cc @@ -48,7 +48,7 @@ std::string formatPerRequestStateParseException(absl::string_view params) { // (["a", "b", "c"]) // There must be at least 2 array elements (a metadata namespace and at least 1 key). std::function -parseUpstreamMetadataField(absl::string_view params_str) { +parseMetadataField(absl::string_view params_str, bool upstream = true) { params_str = StringUtil::trim(params_str); if (params_str.empty() || params_str.front() != '(' || params_str.back() != ')') { throw EnvoyException(formatUpstreamMetadataParseException(params_str)); @@ -72,14 +72,20 @@ parseUpstreamMetadataField(absl::string_view params_str) { throw EnvoyException(formatUpstreamMetadataParseException(params_str)); } - return [params](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { - Upstream::HostDescriptionConstSharedPtr host = stream_info.upstreamHost(); - if (!host) { - return std::string(); + return [upstream, params](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { + const envoy::config::core::v3::Metadata* metadata = nullptr; + if (upstream) { + Upstream::HostDescriptionConstSharedPtr host = stream_info.upstreamHost(); + if (!host) { + return std::string(); + } + metadata = host->metadata().get(); + } else { + metadata = &(stream_info.dynamicMetadata()); } const ProtobufWkt::Value* value = - &::Envoy::Config::Metadata::metadataValue(host->metadata().get(), params[0], params[1]); + &::Envoy::Config::Metadata::metadataValue(metadata, params[0], params[1]); if (value->kind_case() == ProtobufWkt::Value::KIND_NOT_SET) { // No kind indicates default ProtobufWkt::Value which means namespace or key not // found. @@ -339,8 +345,10 @@ StreamInfoHeaderFormatter::StreamInfoHeaderFormatter(absl::string_view field_nam return formatted; }; } else if (absl::StartsWith(field_name, "UPSTREAM_METADATA")) { + field_extractor_ = parseMetadataField(field_name.substr(STATIC_STRLEN("UPSTREAM_METADATA"))); + } else if (absl::StartsWith(field_name, "DYNAMIC_METADATA")) { field_extractor_ = - parseUpstreamMetadataField(field_name.substr(STATIC_STRLEN("UPSTREAM_METADATA"))); + parseMetadataField(field_name.substr(STATIC_STRLEN("DYNAMIC_METADATA")), false); } else if (absl::StartsWith(field_name, "PER_REQUEST_STATE")) { field_extractor_ = parsePerRequestStateField(field_name.substr(STATIC_STRLEN("PER_REQUEST_STATE"))); diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 84140d099d7b..512216c92677 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -615,6 +615,21 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithUpstreamMetadataVariableMiss testFormatting(stream_info, "UPSTREAM_METADATA([\"namespace\", \"key\"])", ""); } +TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithRequestMetadata) { + NiceMock stream_info; + envoy::config::core::v3::Metadata metadata; + ProtobufWkt::Struct struct_obj; + + auto& fields_map = *struct_obj.mutable_fields(); + fields_map["foo"] = ValueUtil::stringValue("bar"); + (*metadata.mutable_filter_metadata())["envoy.lb"] = struct_obj; + + EXPECT_CALL(stream_info, dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); + EXPECT_CALL(Const(stream_info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); + + testFormatting(stream_info, "DYNAMIC_METADATA([\"envoy.lb\", \"foo\"])", "bar"); +} + TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithPerRequestStateVariable) { Envoy::StreamInfo::FilterStateSharedPtr filter_state( std::make_shared( From 8c312f2c5b40b5ce7f6f68a1f9d4b0e98ef3829e Mon Sep 17 00:00:00 2001 From: Kevin Baichoo Date: Thu, 13 Aug 2020 13:11:47 -0400 Subject: [PATCH 54/67] WatchDog Extension hook (#12416) Establish an extension point for actions to run based on Watch Dog Events. Signed-off-by: Kevin Baichoo --- api/envoy/config/bootstrap/v3/bootstrap.proto | 27 +- .../config/bootstrap/v4alpha/bootstrap.proto | 30 +- docs/root/extending/extending.rst | 3 +- docs/root/operations/performance.rst | 11 +- docs/root/version_history/current.rst | 1 + .../envoy/config/bootstrap/v3/bootstrap.proto | 27 +- .../config/bootstrap/v4alpha/bootstrap.proto | 30 +- include/envoy/server/BUILD | 11 + include/envoy/server/configuration.h | 7 + include/envoy/server/guarddog_config.h | 65 +++ source/server/BUILD | 4 + source/server/configuration_impl.cc | 1 + source/server/configuration_impl.h | 6 + source/server/guarddog_impl.cc | 66 ++- source/server/guarddog_impl.h | 12 + test/mocks/server/BUILD | 1 + test/mocks/server/main.cc | 17 +- test/mocks/server/main.h | 7 +- test/server/BUILD | 1 + test/server/guarddog_impl_test.cc | 389 +++++++++++++++++- tools/spelling/spelling_dictionary.txt | 1 + 21 files changed, 693 insertions(+), 24 deletions(-) create mode 100644 include/envoy/server/guarddog_config.h diff --git a/api/envoy/config/bootstrap/v3/bootstrap.proto b/api/envoy/config/bootstrap/v3/bootstrap.proto index 2d096a39c73b..56166456f23f 100644 --- a/api/envoy/config/bootstrap/v3/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3/bootstrap.proto @@ -305,10 +305,35 @@ message ClusterManager { // Envoy process watchdog configuration. When configured, this monitors for // nonresponsive threads and kills the process after the configured thresholds. // See the :ref:`watchdog documentation ` for more information. -// [#next-free-field: 7] +// [#next-free-field: 8] message Watchdog { option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v2.Watchdog"; + message WatchdogAction { + // The events are fired in this order: KILL, MULTIKILL, MEGAMISS, MISS. + // Within an event type, actions execute in the order they are configured. + // For KILL/MULTIKILL there is a default PANIC that will run after the + // registered actions and kills the process if it wasn't already killed. + // It might be useful to specify several debug actions, and possibly an + // alternate FATAL action. + enum WatchdogEvent { + UNKNOWN = 0; + KILL = 1; + MULTIKILL = 2; + MEGAMISS = 3; + MISS = 4; + } + + // Extension specific configuration for the action. + core.v3.TypedExtensionConfig config = 1; + + WatchdogEvent event = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // Register actions that will fire on given WatchDog events. + // See *WatchDogAction* for priority of events. + repeated WatchdogAction actions = 7; + // The duration after which Envoy counts a nonresponsive thread in the // *watchdog_miss* statistic. If not specified the default is 200ms. google.protobuf.Duration miss_timeout = 1; diff --git a/api/envoy/config/bootstrap/v4alpha/bootstrap.proto b/api/envoy/config/bootstrap/v4alpha/bootstrap.proto index ba6107aa8dfe..24faad401e7d 100644 --- a/api/envoy/config/bootstrap/v4alpha/bootstrap.proto +++ b/api/envoy/config/bootstrap/v4alpha/bootstrap.proto @@ -296,10 +296,38 @@ message ClusterManager { // Envoy process watchdog configuration. When configured, this monitors for // nonresponsive threads and kills the process after the configured thresholds. // See the :ref:`watchdog documentation ` for more information. -// [#next-free-field: 7] +// [#next-free-field: 8] message Watchdog { option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v3.Watchdog"; + message WatchdogAction { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.bootstrap.v3.Watchdog.WatchdogAction"; + + // The events are fired in this order: KILL, MULTIKILL, MEGAMISS, MISS. + // Within an event type, actions execute in the order they are configured. + // For KILL/MULTIKILL there is a default PANIC that will run after the + // registered actions and kills the process if it wasn't already killed. + // It might be useful to specify several debug actions, and possibly an + // alternate FATAL action. + enum WatchdogEvent { + UNKNOWN = 0; + KILL = 1; + MULTIKILL = 2; + MEGAMISS = 3; + MISS = 4; + } + + // Extension specific configuration for the action. + core.v4alpha.TypedExtensionConfig config = 1; + + WatchdogEvent event = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // Register actions that will fire on given WatchDog events. + // See *WatchDogAction* for priority of events. + repeated WatchdogAction actions = 7; + // The duration after which Envoy counts a nonresponsive thread in the // *watchdog_miss* statistic. If not specified the default is 200ms. google.protobuf.Duration miss_timeout = 1; diff --git a/docs/root/extending/extending.rst b/docs/root/extending/extending.rst index 316b43547835..ddcd3d874718 100644 --- a/docs/root/extending/extending.rst +++ b/docs/root/extending/extending.rst @@ -19,8 +19,9 @@ types including: * :ref:`Stat sinks ` * :ref:`Tracers ` * :ref:`Request ID ` -* Transport sockets +* :ref:`Transport sockets ` * BoringSSL private key methods +* :ref:`Watchdog action ` * :ref:`Internal redirect policy ` As of this writing there is no high level extension developer documentation. The diff --git a/docs/root/operations/performance.rst b/docs/root/operations/performance.rst index 01acce4acc1f..8846275290da 100644 --- a/docs/root/operations/performance.rst +++ b/docs/root/operations/performance.rst @@ -56,10 +56,13 @@ Watchdog -------- In addition to event loop statistics, Envoy also include a configurable -:ref:`watchdog ` system that can increment -statistics when Envoy is not responsive and optionally kill the server. The statistics are useful -for understanding at a high level whether Envoy's event loop is not responsive either because it is -doing too much work, blocking, or not being scheduled by the OS. +:ref:`watchdog ` +system that can increment statistics when Envoy is not responsive and +optionally kill the server. The system also has an extension point allowing for +custom actions to be taken based on watchdog events. The statistics are +useful for understanding at a high level whether Envoy's event loop is not +responsive either because it is doing too much work, blocking, or not being +scheduled by the OS. The watchdog emits statistics in both the *server.* and *server..* trees. ** is equal to *main_thread*, *worker_0*, *worker_1*, etc. diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index a8a8b0324ff8..052c77e5bdd0 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -76,6 +76,7 @@ New Features * tap: added :ref:`generic body matcher` to scan http requests and responses for text or hex patterns. * tcp: switched the TCP connection pool to the new "shared" connection pool, sharing a common code base with HTTP and HTTP/2. Any unexpected behavioral changes can be temporarily reverted by setting `envoy.reloadable_features.new_tcp_connection_pool` to false. * watchdog: support randomizing the watchdog's kill timeout to prevent synchronized kills via a maximium jitter parameter :ref:`max_kill_timeout_jitter`. +* watchdog: supports an extension point where actions can be registered to fire on watchdog events such as miss, megamiss, kill and multikill. See ref:`watchdog actions`. * xds: added :ref:`extension config discovery` support for HTTP filters. Deprecated diff --git a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto index d3cf6d6947cf..eadc77a8828a 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v3/bootstrap.proto @@ -306,10 +306,35 @@ message ClusterManager { // Envoy process watchdog configuration. When configured, this monitors for // nonresponsive threads and kills the process after the configured thresholds. // See the :ref:`watchdog documentation ` for more information. -// [#next-free-field: 7] +// [#next-free-field: 8] message Watchdog { option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v2.Watchdog"; + message WatchdogAction { + // The events are fired in this order: KILL, MULTIKILL, MEGAMISS, MISS. + // Within an event type, actions execute in the order they are configured. + // For KILL/MULTIKILL there is a default PANIC that will run after the + // registered actions and kills the process if it wasn't already killed. + // It might be useful to specify several debug actions, and possibly an + // alternate FATAL action. + enum WatchdogEvent { + UNKNOWN = 0; + KILL = 1; + MULTIKILL = 2; + MEGAMISS = 3; + MISS = 4; + } + + // Extension specific configuration for the action. + core.v3.TypedExtensionConfig config = 1; + + WatchdogEvent event = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // Register actions that will fire on given WatchDog events. + // See *WatchDogAction* for priority of events. + repeated WatchdogAction actions = 7; + // The duration after which Envoy counts a nonresponsive thread in the // *watchdog_miss* statistic. If not specified the default is 200ms. google.protobuf.Duration miss_timeout = 1; diff --git a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto index 89dd0d7f7d0d..e798d6195f3f 100644 --- a/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto +++ b/generated_api_shadow/envoy/config/bootstrap/v4alpha/bootstrap.proto @@ -304,10 +304,38 @@ message ClusterManager { // Envoy process watchdog configuration. When configured, this monitors for // nonresponsive threads and kills the process after the configured thresholds. // See the :ref:`watchdog documentation ` for more information. -// [#next-free-field: 7] +// [#next-free-field: 8] message Watchdog { option (udpa.annotations.versioning).previous_message_type = "envoy.config.bootstrap.v3.Watchdog"; + message WatchdogAction { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.bootstrap.v3.Watchdog.WatchdogAction"; + + // The events are fired in this order: KILL, MULTIKILL, MEGAMISS, MISS. + // Within an event type, actions execute in the order they are configured. + // For KILL/MULTIKILL there is a default PANIC that will run after the + // registered actions and kills the process if it wasn't already killed. + // It might be useful to specify several debug actions, and possibly an + // alternate FATAL action. + enum WatchdogEvent { + UNKNOWN = 0; + KILL = 1; + MULTIKILL = 2; + MEGAMISS = 3; + MISS = 4; + } + + // Extension specific configuration for the action. + core.v4alpha.TypedExtensionConfig config = 1; + + WatchdogEvent event = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // Register actions that will fire on given WatchDog events. + // See *WatchDogAction* for priority of events. + repeated WatchdogAction actions = 7; + // The duration after which Envoy counts a nonresponsive thread in the // *watchdog_miss* statistic. If not specified the default is 200ms. google.protobuf.Duration miss_timeout = 1; diff --git a/include/envoy/server/BUILD b/include/envoy/server/BUILD index 534270f24e74..34f42371d6ba 100644 --- a/include/envoy/server/BUILD +++ b/include/envoy/server/BUILD @@ -74,6 +74,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "guarddog_config_interface", + hdrs = ["guarddog_config.h"], + deps = [ + ":guarddog_interface", + "//include/envoy/api:api_interface", + "//include/envoy/protobuf:message_validator_interface", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "health_checker_config_interface", hdrs = ["health_checker_config.h"], diff --git a/include/envoy/server/configuration.h b/include/envoy/server/configuration.h index aee4ecf01c04..5fd0a37374f6 100644 --- a/include/envoy/server/configuration.h +++ b/include/envoy/server/configuration.h @@ -73,6 +73,13 @@ class Main { * for at least MultiKillTimeout before we kill the process. */ virtual double wdMultiKillThreshold() const PURE; + + /** + * @return Protobuf::RepeatedPtrField + * the WatchDog Actions that trigger on WatchDog Events. + */ + virtual Protobuf::RepeatedPtrField + wdActions() const PURE; }; /** diff --git a/include/envoy/server/guarddog_config.h b/include/envoy/server/guarddog_config.h new file mode 100644 index 000000000000..3f775d737379 --- /dev/null +++ b/include/envoy/server/guarddog_config.h @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include "envoy/api/api.h" +#include "envoy/common/pure.h" +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/typed_config.h" +#include "envoy/event/dispatcher.h" +#include "envoy/protobuf/message_validator.h" +#include "envoy/server/guarddog.h" + +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Server { +namespace Configuration { + +struct GuardDogActionFactoryContext { + Api::Api& api_; + Event::Dispatcher& dispatcher_; // not owned (this is the guard dog's dispatcher) +}; + +class GuardDogAction { +public: + virtual ~GuardDogAction() = default; + /** + * Callback function for when the GuardDog observes an event. + * @param event the event the GuardDog observes. + * @param thread_ltt_pairs pairs of the relevant thread to the event, and the + * last time touched (LTT) of those threads with their watchdog. + * @param now the current time. + */ + virtual void run(envoy::config::bootstrap::v3::Watchdog::WatchdogAction::WatchdogEvent event, + std::vector> thread_ltt_pairs, + MonotonicTime now) PURE; +}; + +using GuardDogActionPtr = std::unique_ptr; + +/** + * Implemented by each custom GuardDogAction and registered via Registry::registerFactory() + * or the convenience class RegisterFactory. + */ +class GuardDogActionFactory : public Config::TypedFactory { +public: + ~GuardDogActionFactory() override = default; + + /** + * Creates a particular GuardDog Action factory implementation. + * + * @param config supplies the configuration for the action. + * @param context supplies the GuardDog Action's context. + * @return GuardDogActionPtr the GuardDogAction object. + */ + virtual GuardDogActionPtr createGuardDogActionFromProto( + const envoy::config::bootstrap::v3::Watchdog::WatchdogAction& config, + GuardDogActionFactoryContext& context) PURE; + + std::string category() const override { return "envoy.guarddog_actions"; } +}; + +} // namespace Configuration +} // namespace Server +} // namespace Envoy diff --git a/source/server/BUILD b/source/server/BUILD index 7bfcd7699576..c9351c86921e 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -114,6 +114,7 @@ envoy_cc_library( "//include/envoy/common:time_interface", "//include/envoy/event:timer_interface", "//include/envoy/server:configuration_interface", + "//include/envoy/server:guarddog_config_interface", "//include/envoy/server:guarddog_interface", "//include/envoy/server:watchdog_interface", "//include/envoy/stats:stats_interface", @@ -121,8 +122,11 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", "//source/common/common:thread_lib", + "//source/common/config:utility_lib", "//source/common/event:libevent_lib", + "//source/common/protobuf:utility_lib", "//source/common/stats:symbol_table_lib", + "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", ], ) diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc index 7510f068f7ee..44f2a968221a 100644 --- a/source/server/configuration_impl.cc +++ b/source/server/configuration_impl.cc @@ -108,6 +108,7 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(watchdog, multikill_timeout, 0)); watchdog_multikill_threshold_ = PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(watchdog, multikill_threshold, 0.0); + watchdog_actions_ = bootstrap.watchdog().actions(); initializeStatsSinks(bootstrap, server); } diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h index d1c88000c1d1..2de17cff4d47 100644 --- a/source/server/configuration_impl.h +++ b/source/server/configuration_impl.h @@ -111,6 +111,10 @@ class MainImpl : Logger::Loggable, public Main { } double wdMultiKillThreshold() const override { return watchdog_multikill_threshold_; } + Protobuf::RepeatedPtrField + wdActions() const override { + return watchdog_actions_; + } private: /** @@ -129,6 +133,8 @@ class MainImpl : Logger::Loggable, public Main { std::chrono::milliseconds watchdog_kill_timeout_; std::chrono::milliseconds watchdog_multikill_timeout_; double watchdog_multikill_threshold_; + Protobuf::RepeatedPtrField + watchdog_actions_; }; /** diff --git a/source/server/guarddog_impl.cc b/source/server/guarddog_impl.cc index add9ca270d51..cce74e493d9e 100644 --- a/source/server/guarddog_impl.cc +++ b/source/server/guarddog_impl.cc @@ -4,12 +4,21 @@ #include #include +#include +#include +#include "envoy/common/time.h" +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/server/guarddog.h" +#include "envoy/server/guarddog_config.h" #include "envoy/stats/scope.h" #include "common/common/assert.h" #include "common/common/fmt.h" #include "common/common/lock_guard.h" +#include "common/common/logger.h" +#include "common/config/utility.h" +#include "common/protobuf/utility.h" #include "common/stats/symbol_table_impl.h" #include "server/watchdog_impl.h" @@ -42,7 +51,25 @@ GuardDogImpl::GuardDogImpl(Stats::Scope& stats_scope, const Server::Configuratio Stats::StatNameManagedStorage("server.watchdog_mega_miss", stats_scope.symbolTable()) .statName())), dispatcher_(api.allocateDispatcher("guarddog_thread")), - loop_timer_(dispatcher_->createTimer([this]() { step(); })), run_thread_(true) { + loop_timer_(dispatcher_->createTimer([this]() { step(); })), + events_to_actions_([&](const Server::Configuration::Main& config) -> EventToActionsMap { + EventToActionsMap map; + + // We should be able to share the dispatcher since guard dog's lifetime + // should eclipse those of actions. + Configuration::GuardDogActionFactoryContext context = {api, *dispatcher_}; + + const auto& actions = config.wdActions(); + for (const auto& action : actions) { + // Get factory and add the created cb + auto& factory = Config::Utility::getAndCheckFactory( + action.config()); + map[action.event()].push_back(factory.createGuardDogActionFromProto(action, context)); + } + + return map; + }(config)), + run_thread_(true) { start(api); } @@ -61,9 +88,11 @@ void GuardDogImpl::step() { } const auto now = time_source_.monotonicTime(); + std::vector> miss_threads; + std::vector> mega_miss_threads; { - size_t multi_kill_count = 0; + std::vector> multi_kill_threads; Thread::LockGuard guard(wd_lock_); // Compute the multikill threshold @@ -73,6 +102,7 @@ void GuardDogImpl::step() { for (auto& watched_dog : watched_dogs_) { const auto ltt = watched_dog->dog_->lastTouchTime(); + const auto tid = watched_dog->dog_->threadId(); const auto delta = now - ltt; if (watched_dog->last_alert_time_ && watched_dog->last_alert_time_.value() < ltt) { watched_dog->miss_alerted_ = false; @@ -84,6 +114,7 @@ void GuardDogImpl::step() { watched_dog->miss_counter_.inc(); watched_dog->last_alert_time_ = ltt; watched_dog->miss_alerted_ = true; + miss_threads.emplace_back(tid, ltt); } } if (delta > megamiss_timeout_) { @@ -92,22 +123,38 @@ void GuardDogImpl::step() { watched_dog->megamiss_counter_.inc(); watched_dog->last_alert_time_ = ltt; watched_dog->megamiss_alerted_ = true; + mega_miss_threads.emplace_back(tid, ltt); } } if (killEnabled() && delta > kill_timeout_) { + invokeGuardDogActions(WatchDogAction::KILL, {{tid, ltt}}, now); + PANIC(fmt::format("GuardDog: one thread ({}) stuck for more than watchdog_kill_timeout", watched_dog->dog_->threadId().debugString())); } if (multikillEnabled() && delta > multi_kill_timeout_) { - if (++multi_kill_count >= required_for_multi_kill) { + multi_kill_threads.emplace_back(tid, ltt); + + if (multi_kill_threads.size() >= required_for_multi_kill) { + invokeGuardDogActions(WatchDogAction::MULTIKILL, multi_kill_threads, now); + PANIC(fmt::format("GuardDog: At least {} threads ({},...) stuck for more than " "watchdog_multikill_timeout", - multi_kill_count, watched_dog->dog_->threadId().debugString())); + multi_kill_threads.size(), tid.debugString())); } } } } + // Run megamiss and miss handlers + if (!mega_miss_threads.empty()) { + invokeGuardDogActions(WatchDogAction::MEGAMISS, mega_miss_threads, now); + } + + if (!miss_threads.empty()) { + invokeGuardDogActions(WatchDogAction::MISS, miss_threads, now); + } + { Thread::LockGuard guard(mutex_); test_interlock_hook_->signalFromImpl(now); @@ -167,6 +214,17 @@ void GuardDogImpl::stop() { } } +void GuardDogImpl::invokeGuardDogActions( + WatchDogAction::WatchdogEvent event, + std::vector> thread_ltt_pairs, MonotonicTime now) { + const auto& registered_actions = events_to_actions_.find(event); + if (registered_actions != events_to_actions_.end()) { + for (auto& action : registered_actions->second) { + action->run(event, thread_ltt_pairs, now); + } + } +} + GuardDogImpl::WatchedDog::WatchedDog(Stats::Scope& stats_scope, const std::string& thread_name, const WatchDogSharedPtr& watch_dog) : dog_(watch_dog), diff --git a/source/server/guarddog_impl.h b/source/server/guarddog_impl.h index 2fba7f0edcbb..b854215bbbd9 100644 --- a/source/server/guarddog_impl.h +++ b/source/server/guarddog_impl.h @@ -4,9 +4,11 @@ #include #include "envoy/api/api.h" +#include "envoy/config/bootstrap/v3/bootstrap.pb.h" #include "envoy/event/timer.h" #include "envoy/server/configuration.h" #include "envoy/server/guarddog.h" +#include "envoy/server/guarddog_config.h" #include "envoy/server/watchdog.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats.h" @@ -100,6 +102,13 @@ class GuardDogImpl : public GuardDog { bool killEnabled() const { return kill_timeout_ > std::chrono::milliseconds(0); } bool multikillEnabled() const { return multi_kill_timeout_ > std::chrono::milliseconds(0); } + using WatchDogAction = envoy::config::bootstrap::v3::Watchdog::WatchdogAction; + // Helper function to invoke all the GuardDogActions registered for an Event. + void + invokeGuardDogActions(WatchDogAction::WatchdogEvent event, + std::vector> thread_ltt_pairs, + MonotonicTime now); + struct WatchedDog { WatchedDog(Stats::Scope& stats_scope, const std::string& thread_name, const WatchDogSharedPtr& watch_dog); @@ -129,6 +138,9 @@ class GuardDogImpl : public GuardDog { Thread::ThreadPtr thread_; Event::DispatcherPtr dispatcher_; Event::TimerPtr loop_timer_; + using EventToActionsMap = absl::flat_hash_map>; + EventToActionsMap events_to_actions_; Thread::MutexBasicLockable mutex_; bool run_thread_ ABSL_GUARDED_BY(mutex_); }; diff --git a/test/mocks/server/BUILD b/test/mocks/server/BUILD index 35a1ac14b80a..cee8e3a58fb5 100644 --- a/test/mocks/server/BUILD +++ b/test/mocks/server/BUILD @@ -197,6 +197,7 @@ envoy_cc_mock( deps = [ "//include/envoy/server:configuration_interface", "//include/envoy/server:overload_manager_interface", + "//test/test_common:utility_lib", ], ) diff --git a/test/mocks/server/main.cc b/test/mocks/server/main.cc index 26bde5941bed..0b6c739feaad 100644 --- a/test/mocks/server/main.cc +++ b/test/mocks/server/main.cc @@ -1,5 +1,7 @@ #include "main.h" +#include "test/test_common/utility.h" + #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -10,14 +12,25 @@ namespace Configuration { using ::testing::Return; MockMain::MockMain(int wd_miss, int wd_megamiss, int wd_kill, int wd_multikill, - double wd_multikill_threshold) + double wd_multikill_threshold, const std::vector wd_action_protos) : wd_miss_(wd_miss), wd_megamiss_(wd_megamiss), wd_kill_(wd_kill), wd_multikill_(wd_multikill), - wd_multikill_threshold_(wd_multikill_threshold) { + wd_multikill_threshold_(wd_multikill_threshold), wd_actions_([&]() { + Protobuf::RepeatedPtrField actions; + + for (const auto& action_proto_str : wd_action_protos) { + envoy::config::bootstrap::v3::Watchdog::WatchdogAction action; + TestUtility::loadFromJson(action_proto_str, action); + actions.Add()->CopyFrom(action); + } + + return actions; + }()) { ON_CALL(*this, wdMissTimeout()).WillByDefault(Return(wd_miss_)); ON_CALL(*this, wdMegaMissTimeout()).WillByDefault(Return(wd_megamiss_)); ON_CALL(*this, wdKillTimeout()).WillByDefault(Return(wd_kill_)); ON_CALL(*this, wdMultiKillTimeout()).WillByDefault(Return(wd_multikill_)); ON_CALL(*this, wdMultiKillThreshold()).WillByDefault(Return(wd_multikill_threshold_)); + ON_CALL(*this, wdActions).WillByDefault(Return(wd_actions_)); } MockMain::~MockMain() = default; diff --git a/test/mocks/server/main.h b/test/mocks/server/main.h index c89b637e669a..91d2ee19b38e 100644 --- a/test/mocks/server/main.h +++ b/test/mocks/server/main.h @@ -15,9 +15,9 @@ namespace Server { namespace Configuration { class MockMain : public Main { public: - MockMain() : MockMain(0, 0, 0, 0, 0.0) {} + MockMain() : MockMain(0, 0, 0, 0, 0.0, {}) {} MockMain(int wd_miss, int wd_megamiss, int wd_kill, int wd_multikill, - double wd_multikill_threshold); + double wd_multikill_threshold, const std::vector wd_action_protos); ~MockMain() override; MOCK_METHOD(Upstream::ClusterManager*, clusterManager, ()); @@ -28,12 +28,15 @@ class MockMain : public Main { MOCK_METHOD(std::chrono::milliseconds, wdKillTimeout, (), (const)); MOCK_METHOD(std::chrono::milliseconds, wdMultiKillTimeout, (), (const)); MOCK_METHOD(double, wdMultiKillThreshold, (), (const)); + MOCK_METHOD(Protobuf::RepeatedPtrField, + wdActions, (), (const)); std::chrono::milliseconds wd_miss_; std::chrono::milliseconds wd_megamiss_; std::chrono::milliseconds wd_kill_; std::chrono::milliseconds wd_multikill_; double wd_multikill_threshold_; + Protobuf::RepeatedPtrField wd_actions_; }; } // namespace Configuration } // namespace Server diff --git a/test/server/BUILD b/test/server/BUILD index 4718ce6669fb..eb11d33ce7c3 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -136,6 +136,7 @@ envoy_cc_test( "//test/mocks:common_lib", "//test/mocks/server:main_mocks", "//test/mocks/stats:stats_mocks", + "//test/test_common:registry_lib", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", ], diff --git a/test/server/guarddog_impl_test.cc b/test/server/guarddog_impl_test.cc index e26856e011db..c7c090d3faf9 100644 --- a/test/server/guarddog_impl_test.cc +++ b/test/server/guarddog_impl_test.cc @@ -1,19 +1,24 @@ #include #include +#include #include +#include #include "envoy/common/time.h" +#include "envoy/server/guarddog_config.h" #include "envoy/server/watchdog.h" #include "common/api/api_impl.h" #include "common/common/macros.h" #include "common/common/utility.h" +#include "common/protobuf/utility.h" #include "server/guarddog_impl.h" #include "test/mocks/common.h" #include "test/mocks/server/main.h" #include "test/mocks/stats/mocks.h" +#include "test/test_common/registry.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -21,6 +26,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::ElementsAre; using testing::InSequence; using testing::NiceMock; @@ -28,6 +34,15 @@ namespace Envoy { namespace Server { namespace { +// Kill has an explicit value that disables the feature. +const int DISABLE_KILL = 0; +const int DISABLE_MULTIKILL = 0; + +// Miss / Megamiss don't have an explicit value that disables them +// so set a timeout larger than those used in tests for 'disable' it. +const int DISABLE_MISS = 1000000; +const int DISABLE_MEGAMISS = 1000000; + class DebugTestInterlock : public GuardDogImpl::TestInterlockHook { public: // GuardDogImpl::TestInterlockHook @@ -90,8 +105,9 @@ INSTANTIATE_TEST_SUITE_P(TimeSystemType, GuardDogTestBase, class GuardDogDeathTest : public GuardDogTestBase { protected: GuardDogDeathTest() - : config_kill_(1000, 1000, 100, 1000, 0), config_multikill_(1000, 1000, 1000, 500, 0), - config_multikill_threshold_(1000, 1000, 1000, 500, 60) {} + : config_kill_(1000, 1000, 100, 1000, 0, std::vector{}), + config_multikill_(1000, 1000, 1000, 500, 0, std::vector{}), + config_multikill_threshold_(1000, 1000, 1000, 500, 60, std::vector{}) {} /** * This does everything but the final forceCheckForTest() that should cause @@ -255,7 +271,9 @@ TEST_P(GuardDogAlmostDeadTest, NearDeathTest) { class GuardDogMissTest : public GuardDogTestBase { protected: - GuardDogMissTest() : config_miss_(500, 1000, 0, 0, 0), config_mega_(1000, 500, 0, 0, 0) {} + GuardDogMissTest() + : config_miss_(500, 1000, 0, 0, 0, std::vector{}), + config_mega_(1000, 500, 0, 0, 0, std::vector{}) {} void checkMiss(uint64_t count, const std::string& descriptor) { EXPECT_EQ(count, TestUtility::findCounter(stats_store_, "server.watchdog_miss")->value()) @@ -375,27 +393,27 @@ TEST_P(GuardDogMissTest, MissCountTest) { TEST_P(GuardDogTestBase, StartStopTest) { NiceMock stats; - NiceMock config(0, 0, 0, 0, 0); + NiceMock config(0, 0, 0, 0, 0, std::vector{}); initGuardDog(stats, config); } TEST_P(GuardDogTestBase, LoopIntervalNoKillTest) { NiceMock stats; - NiceMock config(40, 50, 0, 0, 0); + NiceMock config(40, 50, 0, 0, 0, std::vector{}); initGuardDog(stats, config); EXPECT_EQ(guard_dog_->loopIntervalForTest(), std::chrono::milliseconds(40)); } TEST_P(GuardDogTestBase, LoopIntervalTest) { NiceMock stats; - NiceMock config(100, 90, 1000, 500, 0); + NiceMock config(100, 90, 1000, 500, 0, std::vector{}); initGuardDog(stats, config); EXPECT_EQ(guard_dog_->loopIntervalForTest(), std::chrono::milliseconds(90)); } TEST_P(GuardDogTestBase, WatchDogThreadIdTest) { NiceMock stats; - NiceMock config(100, 90, 1000, 500, 0); + NiceMock config(100, 90, 1000, 500, 0, std::vector{}); initGuardDog(stats, config); auto watched_dog = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); @@ -415,6 +433,363 @@ TEST_P(GuardDogTestBase, AtomicIsAtomicTest) { ASSERT_EQ(atomic_time.is_lock_free(), true); } +// A GuardDogAction used for testing the GuardDog. +// It's primary use is dumping string of the format EVENT_TYPE : tid1,.., tidN to +// the events vector passed to it. +// Instances of this class will be registered for GuardDogEvent through +// TestGuardDogActionFactory. +class RecordGuardDogAction : public Configuration::GuardDogAction { +public: + RecordGuardDogAction(std::vector& events) : events_(events) {} + + void run(envoy::config::bootstrap::v3::Watchdog::WatchdogAction::WatchdogEvent event, + std::vector> thread_ltt_pairs, + MonotonicTime /*now*/) override { + std::string event_string = + envoy::config::bootstrap::v3::Watchdog::WatchdogAction::WatchdogEvent_Name(event); + absl::StrAppend(&event_string, " : "); + std::vector output_string_parts; + output_string_parts.reserve(thread_ltt_pairs.size()); + + for (const auto& thread_ltt_pair : thread_ltt_pairs) { + output_string_parts.push_back(thread_ltt_pair.first.debugString()); + } + + absl::StrAppend(&event_string, absl::StrJoin(output_string_parts, ",")); + events_.push_back(event_string); + } + +protected: + std::vector& events_; // not owned +}; + +// A GuardDogAction that raises the specified signal. +class AssertGuardDogAction : public Configuration::GuardDogAction { +public: + AssertGuardDogAction() = default; + + void run(envoy::config::bootstrap::v3::Watchdog::WatchdogAction::WatchdogEvent /*event*/, + std::vector> /*thread_ltt_pairs*/, + MonotonicTime /*now*/) override { + RELEASE_ASSERT(false, "ASSERT_GUARDDOG_ACTION"); + } +}; + +// Test factory for consuming Watchdog configs and creating GuardDogActions. +template +class RecordGuardDogActionFactory : public Configuration::GuardDogActionFactory { +public: + RecordGuardDogActionFactory(const std::string& name, std::vector& events) + : name_(name), events_(events) {} + + Configuration::GuardDogActionPtr createGuardDogActionFromProto( + const envoy::config::bootstrap::v3::Watchdog::WatchdogAction& /*config*/, + Configuration::GuardDogActionFactoryContext& /*context*/) override { + // Return different actions depending on the config. + return std::make_unique(events_); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new ConfigType()}; + } + + std::string name() const override { return name_; } + + const std::string name_; + std::vector& events_; // not owned +}; + +// Test factory for consuming Watchdog configs and creating GuardDogActions. +template +class AssertGuardDogActionFactory : public Configuration::GuardDogActionFactory { +public: + AssertGuardDogActionFactory(const std::string& name) : name_(name) {} + + Configuration::GuardDogActionPtr createGuardDogActionFromProto( + const envoy::config::bootstrap::v3::Watchdog::WatchdogAction& /*config*/, + Configuration::GuardDogActionFactoryContext& /*context*/) override { + // Return different actions depending on the config. + return std::make_unique(); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new ConfigType()}; + } + + std::string name() const override { return name_; } + + const std::string name_; +}; + +/** + * Tests that various actions registered for the guard dog get called upon. + */ +class GuardDogActionsTest : public GuardDogTestBase { +protected: + GuardDogActionsTest() + : log_factory_("LogFactory", events_), register_log_factory_(log_factory_), + assert_factory_("AssertFactory"), register_assert_factory_(assert_factory_) {} + + std::vector getActionsConfig() { + return { + R"EOF( + { + "config": { + "name": "AssertFactory", + "typed_config": { + "@type": "type.googleapis.com/google.protobuf.Empty" + } + }, + "event": "MULTIKILL" + } + )EOF", + R"EOF( + { + "config": { + "name": "AssertFactory", + "typed_config": { + "@type": "type.googleapis.com/google.protobuf.Empty" + } + }, + "event": "KILL" + } + )EOF", + R"EOF( + { + "config": { + "name": "LogFactory", + "typed_config": { + "@type": "type.googleapis.com/google.protobuf.Empty" + } + }, + "event": "MEGAMISS" + } + )EOF", + R"EOF( + { + "config": { + "name": "LogFactory", + "typed_config": { + "@type": "type.googleapis.com/google.protobuf.Empty" + } + }, + "event": "MISS" + } + )EOF"}; + } + + void setupFirstDog(const NiceMock& config) { + initGuardDog(fake_stats_, config); + first_dog_ = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + guard_dog_->forceCheckForTest(); + } + + std::vector actions_; + std::vector events_; + RecordGuardDogActionFactory log_factory_; + Registry::InjectFactory register_log_factory_; + AssertGuardDogActionFactory assert_factory_; + Registry::InjectFactory register_assert_factory_; + NiceMock fake_stats_; + WatchDogSharedPtr first_dog_; + WatchDogSharedPtr second_dog_; +}; + +INSTANTIATE_TEST_SUITE_P(TimeSystemType, GuardDogActionsTest, + testing::ValuesIn({TimeSystemType::Real, TimeSystemType::Simulated})); + +TEST_P(GuardDogActionsTest, MissShouldOnlyReportRelevantThreads) { + const NiceMock config(100, DISABLE_MEGAMISS, DISABLE_KILL, + DISABLE_MULTIKILL, 0, getActionsConfig()); + setupFirstDog(config); + second_dog_ = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + time_system_->advanceTimeWait(std::chrono::milliseconds(99)); + second_dog_->touch(); + + time_system_->advanceTimeWait(std::chrono::milliseconds(2)); + guard_dog_->forceCheckForTest(); + + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MISS : ", api_->threadFactory().currentThreadId().debugString()))); +} + +TEST_P(GuardDogActionsTest, MissShouldBeAbleToReportMultipleThreads) { + const NiceMock config(100, DISABLE_MEGAMISS, DISABLE_KILL, + DISABLE_MULTIKILL, 0, getActionsConfig()); + initGuardDog(fake_stats_, config); + first_dog_ = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + second_dog_ = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + + first_dog_->touch(); + second_dog_->touch(); + + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MISS : ", api_->threadFactory().currentThreadId().debugString(), ",", + api_->threadFactory().currentThreadId().debugString()))); +} + +TEST_P(GuardDogActionsTest, MissShouldSaturateOnMissEvent) { + const NiceMock config(100, DISABLE_MISS, DISABLE_KILL, DISABLE_MULTIKILL, + 0, getActionsConfig()); + setupFirstDog(config); + + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MISS : ", api_->threadFactory().currentThreadId().debugString()))); + + // Should saturate and not add an additional "event_" + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MISS : ", api_->threadFactory().currentThreadId().debugString()))); + + // Touch the watchdog, which should allow the event to trigger again. + first_dog_->touch(); + + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT( + events_, + ElementsAre(absl::StrCat("MISS : ", api_->threadFactory().currentThreadId().debugString()), + absl::StrCat("MISS : ", api_->threadFactory().currentThreadId().debugString()))); +} + +TEST_P(GuardDogActionsTest, MegaMissShouldOnlyReportRelevantThreads) { + const NiceMock config(DISABLE_MISS, 100, DISABLE_KILL, DISABLE_MULTIKILL, + 0, getActionsConfig()); + setupFirstDog(config); + second_dog_ = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + time_system_->advanceTimeWait(std::chrono::milliseconds(99)); + second_dog_->touch(); + + time_system_->advanceTimeWait(std::chrono::milliseconds(2)); + guard_dog_->forceCheckForTest(); + + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MEGAMISS : ", api_->threadFactory().currentThreadId().debugString()))); +} + +TEST_P(GuardDogActionsTest, MegaMissShouldBeAbleToReportMultipleThreads) { + const NiceMock config(DISABLE_MISS, 100, DISABLE_KILL, DISABLE_MULTIKILL, + 0, getActionsConfig()); + initGuardDog(fake_stats_, config); + first_dog_ = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + second_dog_ = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + + first_dog_->touch(); + second_dog_->touch(); + + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MEGAMISS : ", api_->threadFactory().currentThreadId().debugString(), + ",", api_->threadFactory().currentThreadId().debugString()))); +} + +TEST_P(GuardDogActionsTest, MegaMissShouldSaturateOnMegaMissEvent) { + const NiceMock config(DISABLE_MISS, 100, DISABLE_KILL, DISABLE_MULTIKILL, + 0, getActionsConfig()); + setupFirstDog(config); + + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MEGAMISS : ", api_->threadFactory().currentThreadId().debugString()))); + + // Should saturate and not add an additional "event_" + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT(events_, ElementsAre(absl::StrCat( + "MEGAMISS : ", api_->threadFactory().currentThreadId().debugString()))); + + // Touch the watchdog, which should allow the event to trigger again. + first_dog_->touch(); + + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT( + events_, + ElementsAre( + absl::StrCat("MEGAMISS : ", api_->threadFactory().currentThreadId().debugString()), + absl::StrCat("MEGAMISS : ", api_->threadFactory().currentThreadId().debugString()))); +} + +TEST_P(GuardDogActionsTest, ShouldRespectEventPriority) { + // Priority of events are KILL, MULTIKILL, MEGAMISS and MISS + + // Kill event should fire before the others + auto kill_function = [&]() -> void { + const NiceMock config(100, 100, 100, 100, 0, getActionsConfig()); + initGuardDog(fake_stats_, config); + auto first_dog = + guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + auto second_dog = + guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + }; + + // We expect only the kill action to have fired + EXPECT_DEATH(kill_function(), "ASSERT_GUARDDOG_ACTION"); + + // Multikill event should fire before the others + auto multikill_function = [&]() -> void { + const NiceMock config(100, 100, DISABLE_KILL, 100, 0, + getActionsConfig()); + initGuardDog(fake_stats_, config); + auto first_dog = + guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + auto second_dog = + guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + }; + + EXPECT_DEATH(multikill_function(), "ASSERT_GUARDDOG_ACTION"); + + // We expect megamiss to fire before miss + const NiceMock config(100, 100, DISABLE_KILL, DISABLE_MULTIKILL, 0, + getActionsConfig()); + setupFirstDog(config); + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + EXPECT_THAT( + events_, + ElementsAre( + absl::StrCat("MEGAMISS : ", api_->threadFactory().currentThreadId().debugString()), + absl::StrCat("MISS : ", api_->threadFactory().currentThreadId().debugString()))); +} + +TEST_P(GuardDogActionsTest, KillShouldTriggerGuardDogActions) { + auto die_function = [&]() -> void { + const NiceMock config(DISABLE_MISS, DISABLE_MEGAMISS, 100, 0, 0, + getActionsConfig()); + setupFirstDog(config); + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + }; + + EXPECT_DEATH(die_function(), "ASSERT_GUARDDOG_ACTION"); +} + +TEST_P(GuardDogActionsTest, MultikillShouldTriggerGuardDogActions) { + auto die_function = [&]() -> void { + const NiceMock config(DISABLE_MISS, DISABLE_MEGAMISS, DISABLE_KILL, + 100, 0, getActionsConfig()); + setupFirstDog(config); + second_dog_ = + guard_dog_->createWatchDog(api_->threadFactory().currentThreadId(), "test_thread"); + guard_dog_->forceCheckForTest(); + time_system_->advanceTimeWait(std::chrono::milliseconds(101)); + guard_dog_->forceCheckForTest(); + }; + + EXPECT_DEATH(die_function(), "ASSERT_GUARDDOG_ACTION"); +} + } // namespace } // namespace Server } // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 1d74fa10a296..1caf69b4da80 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -26,6 +26,7 @@ CAS CB CDS CEL +LTT ceil CHACHA CHLO From 466245b50e6d475be841e1c8d5f254dd6ea41272 Mon Sep 17 00:00:00 2001 From: Konstantin Belyalov Date: Thu, 13 Aug 2020 11:42:35 -0600 Subject: [PATCH 55/67] runtime: debug log that condition is always true when fractionalPercent numerator > denominator (#12068) Signed-off-by: Konstantin Belyalov --- source/common/runtime/runtime_impl.cc | 12 ++++++++++++ test/common/runtime/BUILD | 1 + test/common/runtime/runtime_impl_test.cc | 20 ++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 86c59606a80e..befad79c2b4c 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -122,6 +122,18 @@ bool SnapshotImpl::featureEnabled(absl::string_view key, percent = default_value; } + // When numerator > denominator condition is always evaluates to TRUE + // It becomes hard to debug why configuration does not work in case of wrong numerator. + // Log debug message that numerator is invalid. + uint64_t denominator_value = + ProtobufPercentHelper::fractionalPercentDenominatorToInt(percent.denominator()); + if (percent.numerator() > denominator_value) { + ENVOY_LOG(debug, + "WARNING runtime key '{}': numerator ({}) > denominator ({}), condition always " + "evaluates to true", + key, percent.numerator(), denominator_value); + } + return ProtobufPercentHelper::evaluateFractionalPercent(percent, random_value); } diff --git a/test/common/runtime/BUILD b/test/common/runtime/BUILD index 8cb4a424daa7..151b3d071eb7 100644 --- a/test/common/runtime/BUILD +++ b/test/common/runtime/BUILD @@ -57,6 +57,7 @@ envoy_cc_test( "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:environment_lib", + "//test/test_common:logging_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index dad18c5d2bf8..d300bd634e48 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -23,6 +23,7 @@ #include "test/mocks/thread_local/mocks.h" #include "test/mocks/upstream/mocks.h" #include "test/test_common/environment.h" +#include "test/test_common/logging.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -684,6 +685,25 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); } +TEST_F(StaticLoaderImplTest, InvalidNumerator) { + base_ = TestUtility::parseYaml(R"EOF( + invalid_numerator: + numerator: 111 + denominator: HUNDRED + )EOF"); + setup(); + + envoy::type::v3::FractionalPercent fractional_percent; + + // There is no assertion here - when numerator is invalid + // featureEnabled() will just drop debug log line. + EXPECT_CALL(generator_, random()).WillOnce(Return(500000)); + EXPECT_LOG_CONTAINS("debug", + "runtime key 'invalid_numerator': numerator (111) > denominator (100), " + "condition always evaluates to true", + loader_->snapshot().featureEnabled("invalid_numerator", fractional_percent)); +} + TEST_F(StaticLoaderImplTest, RuntimeFromNonWorkerThreads) { // Force the thread to be considered a non-worker thread. tls_.registered_ = false; From cf9b494e75bd8179a04067854fadf676b000262b Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Thu, 13 Aug 2020 15:58:06 -0700 Subject: [PATCH 56/67] network: add tcp listener backlog config (#12625) Signed-off-by: Florin Coras --- api/envoy/config/listener/v3/listener.proto | 6 ++- .../config/listener/v4alpha/listener.proto | 6 ++- .../envoy/config/listener/v3/listener.proto | 6 ++- .../config/listener/v4alpha/listener.proto | 6 ++- include/envoy/common/platform.h | 8 ++++ include/envoy/event/dispatcher.h | 5 ++- include/envoy/network/listener.h | 5 +++ source/common/event/dispatcher_impl.cc | 5 ++- source/common/event/dispatcher_impl.h | 3 +- source/common/network/listener_impl.cc | 8 ++-- source/common/network/listener_impl.h | 3 +- source/server/admin/admin.h | 1 + source/server/config_validation/dispatcher.cc | 3 +- source/server/config_validation/dispatcher.h | 2 +- source/server/connection_handler_impl.cc | 2 +- source/server/listener_impl.cc | 4 ++ source/server/listener_impl.h | 2 + test/common/http/codec_client_test.cc | 3 +- test/common/network/connection_impl_test.cc | 9 +++-- test/common/network/dns_impl_test.cc | 2 +- test/common/network/listener_impl_test.cc | 10 +++-- .../proxy_protocol_regression_test.cc | 1 + .../proxy_protocol/proxy_protocol_test.cc | 2 + .../transport_sockets/tls/ssl_socket_test.cc | 39 ++++++++++++------- test/integration/fake_upstream.h | 1 + test/mocks/event/mocks.h | 7 ++-- test/mocks/network/mocks.h | 1 + test/server/connection_handler_test.cc | 33 +++++++++++++--- test/server/listener_manager_impl_test.cc | 17 ++++++++ 29 files changed, 151 insertions(+), 49 deletions(-) diff --git a/api/envoy/config/listener/v3/listener.proto b/api/envoy/config/listener/v3/listener.proto index 8c5066909caf..88e8ae4ad5b1 100644 --- a/api/envoy/config/listener/v3/listener.proto +++ b/api/envoy/config/listener/v3/listener.proto @@ -36,7 +36,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 24] +// [#next-free-field: 25] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener"; @@ -259,4 +259,8 @@ message Listener { // If not present, treat it as "udp_default_writer". // [#not-implemented-hide:] core.v3.TypedExtensionConfig udp_writer_config = 23; + + // The maximum length a tcp listener's pending connections queue can grow to. If no value is + // provided net.core.somaxconn will be used on Linux and 128 otherwise. + google.protobuf.UInt32Value tcp_backlog_size = 24; } diff --git a/api/envoy/config/listener/v4alpha/listener.proto b/api/envoy/config/listener/v4alpha/listener.proto index c188ecb24490..753f6d733cc0 100644 --- a/api/envoy/config/listener/v4alpha/listener.proto +++ b/api/envoy/config/listener/v4alpha/listener.proto @@ -39,7 +39,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 24] +// [#next-free-field: 25] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.config.listener.v3.Listener"; @@ -262,4 +262,8 @@ message Listener { // If not present, treat it as "udp_default_writer". // [#not-implemented-hide:] core.v4alpha.TypedExtensionConfig udp_writer_config = 23; + + // The maximum length a tcp listener's pending connections queue can grow to. If no value is + // provided net.core.somaxconn will be used on Linux and 128 otherwise. + google.protobuf.UInt32Value tcp_backlog_size = 24; } diff --git a/generated_api_shadow/envoy/config/listener/v3/listener.proto b/generated_api_shadow/envoy/config/listener/v3/listener.proto index 0d0dc5d817a9..d57b12950535 100644 --- a/generated_api_shadow/envoy/config/listener/v3/listener.proto +++ b/generated_api_shadow/envoy/config/listener/v3/listener.proto @@ -36,7 +36,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 24] +// [#next-free-field: 25] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.Listener"; @@ -258,5 +258,9 @@ message Listener { // [#not-implemented-hide:] core.v3.TypedExtensionConfig udp_writer_config = 23; + // The maximum length a tcp listener's pending connections queue can grow to. If no value is + // provided net.core.somaxconn will be used on Linux and 128 otherwise. + google.protobuf.UInt32Value tcp_backlog_size = 24; + google.protobuf.BoolValue hidden_envoy_deprecated_use_original_dst = 4 [deprecated = true]; } diff --git a/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto b/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto index c188ecb24490..753f6d733cc0 100644 --- a/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto +++ b/generated_api_shadow/envoy/config/listener/v4alpha/listener.proto @@ -39,7 +39,7 @@ message ListenerCollection { udpa.core.v1.CollectionEntry entries = 1; } -// [#next-free-field: 24] +// [#next-free-field: 25] message Listener { option (udpa.annotations.versioning).previous_message_type = "envoy.config.listener.v3.Listener"; @@ -262,4 +262,8 @@ message Listener { // If not present, treat it as "udp_default_writer". // [#not-implemented-hide:] core.v4alpha.TypedExtensionConfig udp_writer_config = 23; + + // The maximum length a tcp listener's pending connections queue can grow to. If no value is + // provided net.core.somaxconn will be used on Linux and 128 otherwise. + google.protobuf.UInt32Value tcp_backlog_size = 24; } diff --git a/include/envoy/common/platform.h b/include/envoy/common/platform.h index 71e0795c9a55..7d239287edb9 100644 --- a/include/envoy/common/platform.h +++ b/include/envoy/common/platform.h @@ -273,3 +273,11 @@ struct mmsghdr { #undef SUPPORTS_PTHREAD_NAMING #define SUPPORTS_PTHREAD_NAMING 1 #endif // defined(__ANDROID_API__) + +#if defined(__linux__) +// On Linux, default listen backlog size to net.core.somaxconn which is runtime configurable +#define ENVOY_TCP_BACKLOG_SIZE -1 +#else +// On non-Linux platforms use 128 which is libevent listener default +#define ENVOY_TCP_BACKLOG_SIZE 128 +#endif \ No newline at end of file diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index eca836980102..98bb16ea7c38 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -148,11 +148,12 @@ class Dispatcher { * @param socket supplies the socket to listen on. * @param cb supplies the callbacks to invoke for listener events. * @param bind_to_port controls whether the listener binds to a transport port or not. + * @param backlog_size controls listener pending connections backlog * @return Network::ListenerPtr a new listener that is owned by the caller. */ virtual Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::ListenerCallbacks& cb, - bool bind_to_port) PURE; + Network::ListenerCallbacks& cb, bool bind_to_port, + uint32_t backlog_size) PURE; /** * Creates a logical udp listener on a specific port. diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 3d8257e69c5f..d07e9d01d0b4 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -161,6 +161,11 @@ class ListenerConfig { * @return std::vector access logs emitted by the listener. */ virtual const std::vector& accessLogs() const PURE; + + /** + * @return pending connection backlog for TCP listeners. + */ + virtual uint32_t tcpBacklogSize() const PURE; }; /** diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 76f4a109039f..09827d0ed891 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -138,9 +138,10 @@ Filesystem::WatcherPtr DispatcherImpl::createFilesystemWatcher() { Network::ListenerPtr DispatcherImpl::createListener(Network::SocketSharedPtr&& socket, Network::ListenerCallbacks& cb, - bool bind_to_port) { + bool bind_to_port, uint32_t backlog_size) { ASSERT(isThreadSafe()); - return std::make_unique(*this, std::move(socket), cb, bind_to_port); + return std::make_unique(*this, std::move(socket), cb, bind_to_port, + backlog_size); } Network::UdpListenerPtr DispatcherImpl::createUdpListener(Network::SocketSharedPtr&& socket, diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 0db663dd985b..f16abc2c420d 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -60,7 +60,8 @@ class DispatcherImpl : Logger::Loggable, uint32_t events) override; Filesystem::WatcherPtr createFilesystemWatcher() override; Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::ListenerCallbacks& cb, bool bind_to_port) override; + Network::ListenerCallbacks& cb, bool bind_to_port, + uint32_t backlog_size) override; Network::UdpListenerPtr createUdpListener(Network::SocketSharedPtr&& socket, Network::UdpListenerCallbacks& cb) override; TimerPtr createTimer(TimerCb cb) override; diff --git a/source/common/network/listener_impl.cc b/source/common/network/listener_impl.cc index ebf2f2b7731d..c1722c88239c 100644 --- a/source/common/network/listener_impl.cc +++ b/source/common/network/listener_impl.cc @@ -91,9 +91,7 @@ void ListenerImpl::onSocketEvent(short flags) { } void ListenerImpl::setupServerSocket(Event::DispatcherImpl& dispatcher, Socket& socket) { - // TODO(fcoras): make listen backlog configurable. For now use 128, which is what libevent - // defaults to for listeners configured with a negative (unspecified) backlog - socket.ioHandle().listen(128); + socket.ioHandle().listen(backlog_size_); // Although onSocketEvent drains to completion, use level triggered mode to avoid potential // loss of the trigger due to transient accept errors. @@ -109,8 +107,8 @@ void ListenerImpl::setupServerSocket(Event::DispatcherImpl& dispatcher, Socket& } ListenerImpl::ListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket, - ListenerCallbacks& cb, bool bind_to_port) - : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb) { + ListenerCallbacks& cb, bool bind_to_port, uint32_t backlog_size) + : BaseListenerImpl(dispatcher, std::move(socket)), cb_(cb), backlog_size_(backlog_size) { if (bind_to_port) { setupServerSocket(dispatcher, *socket_); } diff --git a/source/common/network/listener_impl.h b/source/common/network/listener_impl.h index e26eafebe5d2..61ca54d9d14e 100644 --- a/source/common/network/listener_impl.h +++ b/source/common/network/listener_impl.h @@ -15,7 +15,7 @@ namespace Network { class ListenerImpl : public BaseListenerImpl { public: ListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket, ListenerCallbacks& cb, - bool bind_to_port); + bool bind_to_port, uint32_t backlog_size); void disable() override; void enable() override; @@ -25,6 +25,7 @@ class ListenerImpl : public BaseListenerImpl { void setupServerSocket(Event::DispatcherImpl& dispatcher, Socket& socket); ListenerCallbacks& cb_; + const uint32_t backlog_size_; private: void onSocketEvent(short flags); diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index f2446779f952..9d8c587eda42 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -390,6 +390,7 @@ class AdminImpl : public Admin, const std::vector& accessLogs() const override { return empty_access_logs_; } + uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } AdminImpl& parent_; const std::string name_; diff --git a/source/server/config_validation/dispatcher.cc b/source/server/config_validation/dispatcher.cc index 1a75abe41935..91bb1dc66d62 100644 --- a/source/server/config_validation/dispatcher.cc +++ b/source/server/config_validation/dispatcher.cc @@ -22,7 +22,8 @@ Network::DnsResolverSharedPtr ValidationDispatcher::createDnsResolver( } Network::ListenerPtr ValidationDispatcher::createListener(Network::SocketSharedPtr&&, - Network::ListenerCallbacks&, bool) { + Network::ListenerCallbacks&, bool, + uint32_t) { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } diff --git a/source/server/config_validation/dispatcher.h b/source/server/config_validation/dispatcher.h index e322e3f1cdc1..d61d3bcc0b43 100644 --- a/source/server/config_validation/dispatcher.h +++ b/source/server/config_validation/dispatcher.h @@ -27,7 +27,7 @@ class ValidationDispatcher : public DispatcherImpl { createDnsResolver(const std::vector& resolvers, const bool use_tcp_for_dns_lookups) override; Network::ListenerPtr createListener(Network::SocketSharedPtr&&, Network::ListenerCallbacks&, - bool bind_to_port) override; + bool bind_to_port, uint32_t backlog_size) override; protected: std::shared_ptr dns_resolver_{ diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 323ddf4df430..c06545703508 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -141,7 +141,7 @@ ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImp : ActiveTcpListener( parent, parent.dispatcher_.createListener(config.listenSocketFactory().getListenSocket(), *this, - config.bindToPort()), + config.bindToPort(), config.tcpBacklogSize()), config) {} ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImpl& parent, diff --git a/source/server/listener_impl.cc b/source/server/listener_impl.cc index f3fe37ade187..32e3a3efa0c4 100644 --- a/source/server/listener_impl.cc +++ b/source/server/listener_impl.cc @@ -233,6 +233,8 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, per_connection_buffer_limit_bytes, 1024 * 1024)), listener_tag_(parent_.factory_.nextListenerTag()), name_(name), added_via_api_(added_via_api), workers_started_(workers_started), hash_(hash), + tcp_backlog_size_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, tcp_backlog_size, ENVOY_TCP_BACKLOG_SIZE)), validation_visitor_( added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor() : parent_.server_.messageValidationContext().staticValidationVisitor()), @@ -310,6 +312,8 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin, PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, per_connection_buffer_limit_bytes, 1024 * 1024)), listener_tag_(origin.listener_tag_), name_(name), added_via_api_(added_via_api), workers_started_(workers_started), hash_(hash), + tcp_backlog_size_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, tcp_backlog_size, ENVOY_TCP_BACKLOG_SIZE)), validation_visitor_( added_via_api_ ? parent_.server_.messageValidationContext().dynamicValidationVisitor() : parent_.server_.messageValidationContext().staticValidationVisitor()), diff --git a/source/server/listener_impl.h b/source/server/listener_impl.h index 920f8a24e9b3..818056dfdbec 100644 --- a/source/server/listener_impl.h +++ b/source/server/listener_impl.h @@ -311,6 +311,7 @@ class ListenerImpl final : public Network::ListenerConfig, const std::vector& accessLogs() const override { return access_logs_; } + uint32_t tcpBacklogSize() const override { return tcp_backlog_size_; } Init::Manager& initManager(); envoy::config::core::v3::TrafficDirection direction() const override { return config().traffic_direction(); @@ -371,6 +372,7 @@ class ListenerImpl final : public Network::ListenerConfig, const bool added_via_api_; const bool workers_started_; const uint64_t hash_; + const uint32_t tcp_backlog_size_; ProtobufMessage::ValidationVisitor& validation_visitor_; // A target is added to Server's InitManager if workers_started_ is false. diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 15979e8350b3..0fa919d17833 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -286,7 +286,8 @@ class CodecNetworkTest : public testing::TestWithParamcreateClientConnection( socket->localAddress(), source_address_, Network::Test::createRawBufferSocket(), nullptr); - upstream_listener_ = dispatcher_->createListener(std::move(socket), listener_callbacks_, true); + upstream_listener_ = dispatcher_->createListener(std::move(socket), listener_callbacks_, true, + ENVOY_TCP_BACKLOG_SIZE); client_connection_ = client_connection.get(); client_connection_->addConnectionCallbacks(client_callbacks_); diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 0a86156963a0..72c8f691213f 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -108,7 +108,8 @@ class ConnectionImplTest : public testing::TestWithParam { } socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = + dispatcher_->createListener(socket_, listener_callbacks_, true, ENVOY_TCP_BACKLOG_SIZE); client_connection_ = std::make_unique( *dispatcher_, socket_->localAddress(), source_address_, Network::Test::createRawBufferSocket(), socket_options_); @@ -1132,7 +1133,8 @@ TEST_P(ConnectionImplTest, BindFailureTest) { dispatcher_ = api_->allocateDispatcher("test_thread"); socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = + dispatcher_->createListener(socket_, listener_callbacks_, true, ENVOY_TCP_BACKLOG_SIZE); client_connection_ = dispatcher_->createClientConnection( socket_->localAddress(), source_address_, Network::Test::createRawBufferSocket(), nullptr); @@ -2238,7 +2240,8 @@ class ReadBufferLimitTest : public ConnectionImplTest { dispatcher_ = api_->allocateDispatcher("test_thread"); socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = + dispatcher_->createListener(socket_, listener_callbacks_, true, ENVOY_TCP_BACKLOG_SIZE); client_connection_ = dispatcher_->createClientConnection( socket_->localAddress(), Network::Address::InstanceConstSharedPtr(), diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index 8b49d8035bca..678e97852e49 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -437,7 +437,7 @@ class DnsImplTest : public testing::TestWithParam { server_ = std::make_unique(*dispatcher_); socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); - listener_ = dispatcher_->createListener(socket_, *server_, true); + listener_ = dispatcher_->createListener(socket_, *server_, true, ENVOY_TCP_BACKLOG_SIZE); // Point c-ares at the listener with no search domains and TCP-only. peer_ = std::make_unique(dynamic_cast(resolver_.get())); diff --git a/test/common/network/listener_impl_test.cc b/test/common/network/listener_impl_test.cc index 5aaef758ce4a..d3a80fea73c0 100644 --- a/test/common/network/listener_impl_test.cc +++ b/test/common/network/listener_impl_test.cc @@ -34,7 +34,8 @@ static void errorCallbackTest(Address::IpVersion version) { Network::Test::getCanonicalLoopbackAddress(version), nullptr, true); Network::MockListenerCallbacks listener_callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher->createListener(socket, listener_callbacks, true, ENVOY_TCP_BACKLOG_SIZE); Network::ClientConnectionPtr client_connection = dispatcher->createClientConnection( socket->localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -65,8 +66,8 @@ TEST_P(ListenerImplDeathTest, ErrorCallback) { class TestListenerImpl : public ListenerImpl { public: TestListenerImpl(Event::DispatcherImpl& dispatcher, SocketSharedPtr socket, ListenerCallbacks& cb, - bool bind_to_port) - : ListenerImpl(dispatcher, std::move(socket), cb, bind_to_port) {} + bool bind_to_port, uint32_t tcp_backlog = ENVOY_TCP_BACKLOG_SIZE) + : ListenerImpl(dispatcher, std::move(socket), cb, bind_to_port, tcp_backlog) {} MOCK_METHOD(Address::InstanceConstSharedPtr, getLocalAddress, (os_fd_t fd)); }; @@ -150,7 +151,8 @@ TEST_P(ListenerImplTest, GlobalConnectionLimitEnforcement) { Network::Test::getCanonicalLoopbackAddress(version_), nullptr, true); Network::MockListenerCallbacks listener_callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher_->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, listener_callbacks, true, ENVOY_TCP_BACKLOG_SIZE); std::vector client_connections; std::vector server_connections; diff --git a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc index 7c1bd0d80ae1..da6866036fce 100644 --- a/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc +++ b/test/extensions/common/proxy_protocol/proxy_protocol_regression_test.cc @@ -79,6 +79,7 @@ class ProxyProtocolRegressionTest : public testing::TestWithParam& accessLogs() const override { return empty_access_logs_; } + uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } // Network::FilterChainManager const Network::FilterChain* findFilterChain(const Network::ConnectionSocket&) const override { diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index a270ee2f569d..8aa06aad929d 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -94,6 +94,7 @@ class ProxyProtocolTest : public testing::TestWithParam& accessLogs() const override { return empty_access_logs_; } + uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } // Network::FilterChainManager const Network::FilterChain* findFilterChain(const Network::ConnectionSocket&) const override { @@ -1291,6 +1292,7 @@ class WildcardProxyProtocolTest : public testing::TestWithParam& accessLogs() const override { return empty_access_logs_; } + uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } // Network::FilterChainManager const Network::FilterChain* findFilterChain(const Network::ConnectionSocket&) const override { diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 2eb348a432ec..6fa102a74ad4 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -309,7 +309,8 @@ void testUtil(const TestUtilOptions& options) { Network::Test::getCanonicalLoopbackAddress(options.version()), nullptr, true); Network::MockListenerCallbacks callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true); + Network::ListenerPtr listener = + dispatcher->createListener(socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(options.clientCtxYaml()), @@ -610,7 +611,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { Network::Test::getCanonicalLoopbackAddress(options.version()), nullptr, true); NiceMock callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true); + Network::ListenerPtr listener = + dispatcher->createListener(socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); Stats::TestUtil::TestStore client_stats_store; Api::ApiPtr client_api = Api::createApiForTest(client_stats_store, time_system); @@ -2401,7 +2403,8 @@ TEST_P(SslSocketTest, FlushCloseDuringHandshake) { Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); Network::MockListenerCallbacks callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -2457,7 +2460,8 @@ TEST_P(SslSocketTest, HalfClose) { Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); Network::MockListenerCallbacks listener_callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher_->createListener(socket, listener_callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, listener_callbacks, true, ENVOY_TCP_BACKLOG_SIZE); std::shared_ptr server_read_filter(new Network::MockReadFilter()); std::shared_ptr client_read_filter(new Network::MockReadFilter()); @@ -2539,7 +2543,8 @@ TEST_P(SslSocketTest, ClientAuthMultipleCAs) { Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); Network::MockListenerCallbacks callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); const std::string client_ctx_yaml = R"EOF( common_tls_context: @@ -2637,8 +2642,10 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, NiceMock callbacks; Network::MockConnectionHandler connection_handler; Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); - Network::ListenerPtr listener1 = dispatcher->createListener(socket1, callbacks, true); - Network::ListenerPtr listener2 = dispatcher->createListener(socket2, callbacks, true); + Network::ListenerPtr listener1 = + dispatcher->createListener(socket1, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); + Network::ListenerPtr listener2 = + dispatcher->createListener(socket2, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context); @@ -2773,7 +2780,8 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml NiceMock callbacks; Network::MockConnectionHandler connection_handler; Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); - Network::ListenerPtr listener = dispatcher->createListener(tcp_socket, callbacks, true); + Network::ListenerPtr listener = + dispatcher->createListener(tcp_socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext client_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml), client_tls_context); @@ -3215,8 +3223,10 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) { Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); Network::MockListenerCallbacks callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); - Network::ListenerPtr listener2 = dispatcher_->createListener(socket2, callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); + Network::ListenerPtr listener2 = + dispatcher_->createListener(socket2, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); const std::string client_ctx_yaml = R"EOF( common_tls_context: tls_certificates: @@ -3330,7 +3340,8 @@ void SslSocketTest::testClientSessionResumption(const std::string& server_ctx_ya Network::MockConnectionHandler connection_handler; Api::ApiPtr api = Api::createApiForTest(server_stats_store, time_system_); Event::DispatcherPtr dispatcher(server_api->allocateDispatcher("test_thread")); - Network::ListenerPtr listener = dispatcher->createListener(socket, callbacks, true); + Network::ListenerPtr listener = + dispatcher->createListener(socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); Network::ConnectionPtr server_connection; Network::MockConnectionCallbacks server_connection_callbacks; @@ -3590,7 +3601,8 @@ TEST_P(SslSocketTest, SslError) { Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); Network::MockListenerCallbacks callbacks; Network::MockConnectionHandler connection_handler; - Network::ListenerPtr listener = dispatcher_->createListener(socket, callbacks, true); + Network::ListenerPtr listener = + dispatcher_->createListener(socket, callbacks, true, ENVOY_TCP_BACKLOG_SIZE); Network::ClientConnectionPtr client_connection = dispatcher_->createClientConnection( socket->localAddress(), Network::Address::InstanceConstSharedPtr(), @@ -4347,7 +4359,8 @@ class SslReadBufferLimitTest : public SslSocketTest { socket_ = std::make_shared( Network::Test::getCanonicalLoopbackAddress(GetParam()), nullptr, true); - listener_ = dispatcher_->createListener(socket_, listener_callbacks_, true); + listener_ = + dispatcher_->createListener(socket_, listener_callbacks_, true, ENVOY_TCP_BACKLOG_SIZE); TestUtility::loadFromYaml(TestEnvironment::substitute(client_ctx_yaml_), upstream_tls_context_); auto client_cfg = diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 0c23c42e4b41..c9dd430d2440 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -731,6 +731,7 @@ class FakeUpstream : Logger::Loggable, return empty_access_logs_; } ResourceLimit& openConnections() override { return connection_resource_; } + uint32_t tcpBacklogSize() const override { return ENVOY_TCP_BACKLOG_SIZE; } void setMaxConnections(const uint32_t num_connections) { connection_resource_.setMax(num_connections); diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 55983c27c536..187df9f15e8d 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -64,8 +64,9 @@ class MockDispatcher : public Dispatcher { } Network::ListenerPtr createListener(Network::SocketSharedPtr&& socket, - Network::ListenerCallbacks& cb, bool bind_to_port) override { - return Network::ListenerPtr{createListener_(std::move(socket), cb, bind_to_port)}; + Network::ListenerCallbacks& cb, bool bind_to_port, + uint32_t backlog_size) override { + return Network::ListenerPtr{createListener_(std::move(socket), cb, bind_to_port, backlog_size)}; } Network::UdpListenerPtr createUdpListener(Network::SocketSharedPtr&& socket, @@ -115,7 +116,7 @@ class MockDispatcher : public Dispatcher { MOCK_METHOD(Filesystem::Watcher*, createFilesystemWatcher_, ()); MOCK_METHOD(Network::Listener*, createListener_, (Network::SocketSharedPtr && socket, Network::ListenerCallbacks& cb, - bool bind_to_port)); + bool bind_to_port, uint32_t backlog_size)); MOCK_METHOD(Network::UdpListener*, createUdpListener_, (Network::SocketSharedPtr && socket, Network::UdpListenerCallbacks& cb)); MOCK_METHOD(Timer*, createTimer_, (Event::TimerCb cb)); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 144051c0d981..0097df388d7a 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -358,6 +358,7 @@ class MockListenerConfig : public ListenerConfig { MOCK_METHOD(Network::UdpPacketWriterFactoryOptRef, udpPacketWriterFactory, ()); MOCK_METHOD(ConnectionBalancer&, connectionBalancer, ()); MOCK_METHOD(ResourceLimit&, openConnections, ()); + MOCK_METHOD(uint32_t, tcpBacklogSize, (), (const)); envoy::config::core::v3::TrafficDirection direction() const override { return envoy::config::core::v3::UNSPECIFIED; diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index a53c44d6e341..7ee24468032b 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -56,9 +56,11 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> filter_chain_manager = nullptr) + std::shared_ptr> filter_chain_manager = nullptr, + uint32_t tcp_backlog_size = ENVOY_TCP_BACKLOG_SIZE) : parent_(parent), socket_(std::make_shared>()), socket_factory_(std::move(socket_factory)), tag_(tag), bind_to_port_(bind_to_port), + tcp_backlog_size_(tcp_backlog_size), hand_off_restored_destination_connections_(hand_off_restored_destination_connections), name_(name), listener_filters_timeout_(listener_filters_timeout), continue_on_listener_filters_timeout_(continue_on_listener_filters_timeout), @@ -109,6 +111,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable> overridden_filter_chain_manager = - nullptr) { + nullptr, + uint32_t tcp_backlog_size = ENVOY_TCP_BACKLOG_SIZE) { listeners_.emplace_back(std::make_unique( *this, tag, bind_to_port, hand_off_restored_destination_connections, name, socket_type, listener_filters_timeout, continue_on_listener_filters_timeout, socket_factory_, - overridden_filter_chain_manager)); + overridden_filter_chain_manager, tcp_backlog_size)); EXPECT_CALL(*socket_factory_, socketType()).WillOnce(Return(socket_type)); if (listener == nullptr) { // Expecting listener config in place update. @@ -191,10 +196,10 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggablesocket_)); if (socket_type == Network::Socket::Type::Stream) { - EXPECT_CALL(dispatcher_, createListener_(_, _, _)) + EXPECT_CALL(dispatcher_, createListener_(_, _, _, _)) .WillOnce(Invoke([listener, listener_callbacks](Network::SocketSharedPtr&&, - Network::ListenerCallbacks& cb, - bool) -> Network::Listener* { + Network::ListenerCallbacks& cb, bool, + uint32_t) -> Network::Listener* { if (listener_callbacks != nullptr) { *listener_callbacks = &cb; } @@ -1085,6 +1090,22 @@ TEST_F(ConnectionHandlerTest, ShutdownUdpListener) { << "The read_filter_ should be deleted before the udp_listener_ is deleted."; } +TEST_F(ConnectionHandlerTest, TcpBacklogCustom) { + uint32_t custom_backlog = 100; + TestListener* test_listener = addListener( + 1, true, false, "test_tcp_backlog", nullptr, nullptr, nullptr, nullptr, + Network::Socket::Type::Stream, std::chrono::milliseconds(), false, nullptr, custom_backlog); + EXPECT_CALL(*socket_factory_, getListenSocket()).WillOnce(Return(listeners_.back()->socket_)); + EXPECT_CALL(*socket_factory_, localAddress()).WillOnce(ReturnRef(local_address_)); + EXPECT_CALL(dispatcher_, createListener_(_, _, _, _)) + .WillOnce(Invoke([custom_backlog](Network::SocketSharedPtr&&, Network::ListenerCallbacks&, + bool, uint32_t backlog) -> Network::Listener* { + EXPECT_EQ(custom_backlog, backlog); + return nullptr; + })); + handler_->addListener(absl::nullopt, *test_listener); +} + } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index c47c22f44b6e..a1ccd242cdd3 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -4721,6 +4721,23 @@ TEST_F(ListenerManagerImplTest, UdpDefaultWriterConfig) { EXPECT_FALSE(udp_packet_writer->isBatchMode()); } +TEST_F(ListenerManagerImplTest, TcpBacklogCustomConfig) { + const std::string yaml = TestEnvironment::substitute(R"EOF( + name: TcpBacklogConfigListener + address: + socket_address: { address: 127.0.0.1, port_value: 1111 } + tcp_backlog_size: 100 + filter_chains: + - filters: + )EOF", + Network::Address::IpVersion::v4); + + EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, _)); + manager_->addOrUpdateListener(parseListenerFromV3Yaml(yaml), "", true); + EXPECT_EQ(1U, manager_->listeners().size()); + EXPECT_EQ(100U, manager_->listeners().back().get().tcpBacklogSize()); +} + } // namespace } // namespace Server } // namespace Envoy From 7852b9d98e5cc07856f316047091490d315f0332 Mon Sep 17 00:00:00 2001 From: jianwen612 <55008549+jianwen612@users.noreply.github.com> Date: Thu, 13 Aug 2020 19:14:35 -0500 Subject: [PATCH 57/67] [redis_proxy] added a constraint for route.prefix().size() (#12637) Signed-off-by: jianwen --- .../filters/network/redis_proxy/v3/redis_proxy.proto | 2 +- .../filters/network/redis_proxy/v3/redis_proxy.proto | 2 +- ...s_redis_proxy_stackoverflow_with_long_route_prefix | 11 +++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/oss_redis_proxy_stackoverflow_with_long_route_prefix diff --git a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto index 402937fff28f..740095ac5120 100644 --- a/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto +++ b/api/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto @@ -156,7 +156,7 @@ message RedisProxy { // String prefix that must match the beginning of the keys. Envoy will always favor the // longest match. - string prefix = 1; + string prefix = 1 [(validate.rules).string = {max_bytes: 1000}]; // Indicates if the prefix needs to be removed from the key when forwarded. bool remove_prefix = 2; diff --git a/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto b/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto index 0bc52493bb29..a537435a46e9 100644 --- a/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto +++ b/generated_api_shadow/envoy/extensions/filters/network/redis_proxy/v3/redis_proxy.proto @@ -156,7 +156,7 @@ message RedisProxy { // String prefix that must match the beginning of the keys. Envoy will always favor the // longest match. - string prefix = 1; + string prefix = 1 [(validate.rules).string = {max_bytes: 1000}]; // Indicates if the prefix needs to be removed from the key when forwarded. bool remove_prefix = 2; diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/oss_redis_proxy_stackoverflow_with_long_route_prefix b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/oss_redis_proxy_stackoverflow_with_long_route_prefix new file mode 100644 index 000000000000..3d33080e9e8f --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/oss_redis_proxy_stackoverflow_with_long_route_prefix @@ -0,0 +1,11 @@ +config { + name: "envoy.filters.network.redis_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy" + value: "\n\001\026\032\030\n\002\020\004\030\001 \200\372\003*\0002\006\010\200\200\200\340\0038\004@\001 \001*\204\367\013\n\200\003\n\303\001oooooooooooooooooooooooooooooooo\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000ooooooooooo/ooooooooooooooooooooooo\032\010\177\177\177\177\177\177\333\246\"m\ni\363\270\221\233\364\210\267\225\362\251\233\253\360\240\247\240\361\262\222\254\363\254\261\226\361\252\262\275\362\211\220\242\361\247\243\256\363\263\247\233\360\225\255\250\361\231\220\275\363\237\230\266\363\261\260\206\362\201\247\270\362\272\271\251\363\250\233\222\360\251\242\244\363\250\274\263\363\247\230\236\361\222\260\237\363\233\255\222\363\264\271\267\363\265\276\271\361\234\202\215\360\227\216\255/\030\001\"\n\n\010/dev/fd/\"\014\n\002\000\003\022\006\n\004\010.\020\002\"\014\n\010envoy.fi\030\001\"\027\n\013\177\177\177envoy.fi\022\010\n\006\010\262\354\300\341\005\n\361\001\n\002\000\003\020\001\032\010envoy.fi\"\034\n\030envoy.type.v3.Fractional\030\001\">\n<\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\"\024\n\010envoy.fi\022\010\n\002\010.\022\002\000\003\"k\ni\363\270\221\233\364\210\267\225\362\251\233\253\360\240\247\240\361\262\222\254\363\254\261\226\361\252\262\275\362\211\220\242\361\247\243\256\363\263\247\233\360\225\255\250\307\233\360\225\255\250\361\231\220\275\363\237\230\266\363\261\260\206\362\201\247\270\322\272\360\251\242\244\363\250\274\263\363\247\230\236\361\222\260\237\363\233\255\222\363\264\271\267\363\265\276\271\361\234\202\215\360\227\216\255/\n\243\361\013\n\326\357\013\177\177\177envoconfig {\n name: \"envoy.filters.network.dubbo_proxy\"\n typed_config {\n type_url: \"type.googleapis.com/envoy.extensions.filters.network.dubbo_proxy.v3.DubboProxy\"\n value: \"\\n\\001B\\\"\\277\\373\\002\\n\\001]\\\"\\264\\373\\002?\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000,\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\020\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\177\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\024\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\177\\177\\177\\177\\177\\177\\177\\035\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\002\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\004\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\y.fi000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\002\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000#\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\335\\221\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000*\\000*\\000*\\005\\022\\003\\n\\001\\013\"\n }\n}\n\032\010\177\177\177\177\177\177\333\246\"\022\n\010\001\000\000\000\000\000\000\005\022\004\n\002\020\001\030\001\"\004\n\002\000\003\"\005\n\001&\030\001\"!\n\030envoy.type.v3.Frcational\022\005\n\003\010\256\020\"\014\n\010envoy.fi\030\001\"k\ni\363\270\221\233\364\210\267\225\362\251\233\253\360\240\247\240\361\262\222\254\363\254\261\226\361\252\262\275\362\211\220\242\361\247\243\256\363\263\247\233\360\225\255\250\307\233\360\225\255\250\361\231\220\275\363\237\230\266\363\261\260\206\362\201\247\270\322\272\360\251\242\244\363\250\274\263\363\247\230\236\361\222\260\237\363\233\255\222\363\264\271\267\363\265\276\271\361\234\202\215\360\227\216\255/\nb\n\010\177\177\177\177\177\177\333\246\020\001\032!envoy.filters.netwo\322\225\341\233\203bbo_proxy\"\026\n\010envoy.fi\022\010\n\002\010.\022\002\000\003\030\001\"\031\n\013\177\177\177envoy.fi\022\010\n\006\010\262\354\300\341\005\030\001\020\0012\332\357\013\032\326\357\013\177\177\177envoconfig {\n name: \"envoy.filters.network.dubbo_proxy\"\n typed_config {\n type_url: \"type.googleapis.com/envoy.extensions.filters.network.dubbo_proxy.v3.DubboProxy\"\n value: \"\\n\\001B\\\"\\277\\373\\002\\n\\001]\\\"\\264\\373\\002?\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000,\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\020\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\177\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\024\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\177\\177\\177\\177\\177\\177\\177\\035\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\002\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\004\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\y.fi000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\001\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\002\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000#\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\335\\221\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000*\\000*\\000*\\005\\022\\003\\n\\001\\013\"\n }\n}\n:$\032\"mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm" + } +} +actions { + on_new_connection { + } +} From bd2b989c578b2472faaff44902573e5b187f671f Mon Sep 17 00:00:00 2001 From: Martin Matusiak Date: Fri, 14 Aug 2020 10:15:19 +1000 Subject: [PATCH 58/67] router: add new ratelimited retry backoff strategy (#12202) This PR implements a new retry back off strategy that uses values from response headers like Retry-After or X-RateLimit-Reset (the headers are configurable) to decide the back off interval before retrying a request. Signed-off-by: Martin Matusiak --- .../config/route/v3/route_components.proto | 81 ++++++- .../route/v4alpha/route_components.proto | 87 +++++++- .../http/http_filters/router_filter.rst | 9 +- .../cluster_manager/cluster_stats.rst | 2 + .../intro/arch_overview/http/http_routing.rst | 5 +- docs/root/version_history/current.rst | 1 + .../config/route/v3/route_components.proto | 81 ++++++- .../route/v4alpha/route_components.proto | 87 +++++++- include/envoy/router/BUILD | 1 + include/envoy/router/router.h | 37 +++ include/envoy/upstream/upstream.h | 2 + source/common/common/backoff_strategy.cc | 17 +- source/common/common/backoff_strategy.h | 28 ++- source/common/config/datasource.cc | 4 +- source/common/config/grpc_stream.h | 4 +- source/common/config/utility.h | 3 +- source/common/http/async_client_impl.h | 7 + source/common/router/BUILD | 15 ++ source/common/router/config_impl.cc | 16 ++ source/common/router/config_impl.h | 6 + source/common/router/reset_header_parser.cc | 68 ++++++ source/common/router/reset_header_parser.h | 59 +++++ source/common/router/retry_state_impl.cc | 62 +++++- source/common/router/retry_state_impl.h | 12 +- source/common/router/router.cc | 20 +- source/common/router/router.h | 13 +- .../upstream/health_discovery_service.cc | 4 +- test/common/common/backoff_strategy_test.cc | 39 +++- test/common/config/utility_test.cc | 2 +- test/common/router/BUILD | 13 ++ test/common/router/config_impl_test.cc | 81 +++++++ .../common/router/reset_header_parser_test.cc | 210 ++++++++++++++++++ test/common/router/retry_state_impl_test.cc | 127 ++++++++++- test/common/router/router_test.cc | 2 +- .../common/router/router_upstream_log_test.cc | 2 +- .../dns_cache_impl_test.cc | 2 +- test/integration/stats_integration_test.cc | 6 +- test/mocks/router/mocks.h | 8 + 38 files changed, 1154 insertions(+), 69 deletions(-) create mode 100644 source/common/router/reset_header_parser.cc create mode 100644 source/common/router/reset_header_parser.h create mode 100644 test/common/router/reset_header_parser_test.cc diff --git a/api/envoy/config/route/v3/route_components.proto b/api/envoy/config/route/v3/route_components.proto index 3643d449272e..857bb3a6d827 100644 --- a/api/envoy/config/route/v3/route_components.proto +++ b/api/envoy/config/route/v3/route_components.proto @@ -1050,10 +1050,15 @@ message RouteAction { } // HTTP retry :ref:`architecture overview `. -// [#next-free-field: 11] +// [#next-free-field: 12] message RetryPolicy { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RetryPolicy"; + enum ResetHeaderFormat { + SECONDS = 0; + UNIX_TIMESTAMP = 1; + } + message RetryPriority { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RetryPolicy.RetryPriority"; @@ -1104,6 +1109,69 @@ message RetryPolicy { google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; } + message ResetHeader { + string name = 1 + [(validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}]; + + ResetHeaderFormat format = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // A retry back-off strategy that applies when the upstream server rate limits + // the request. + // + // Given this configuration: + // + // .. code-block:: yaml + // + // rate_limited_retry_back_off: + // reset_headers: + // - name: Retry-After + // format: SECONDS + // - name: X-RateLimit-Reset + // format: UNIX_TIMESTAMP + // max_interval: "300s" + // + // The following algorithm will apply: + // + // 1. If the response contains the header ``Retry-After`` its value must be on + // the form ``120`` (an integer that represents the number of seconds to + // wait before retrying). If so, this value is used as the back-off interval. + // 2. Otherwise, if the response contains the header ``X-RateLimit-Reset`` its + // value must be on the form ``1595320702`` (an integer that represents the + // point in time at which to retry, as a Unix timestamp in seconds). If so, + // the current time is subtracted from this value and the result is used as + // the back-off interval. + // 3. Otherwise, Envoy will use the default + // :ref:`exponential back-off ` + // strategy. + // + // No matter which format is used, if the resulting back-off interval exceeds + // ``max_interval`` it is discarded and the next header in ``reset_headers`` + // is tried. If a request timeout is configured for the route it will further + // limit how long the request will be allowed to run. + // + // To prevent many clients retrying at the same point in time jitter is added + // to the back-off interval, so the resulting interval is decided by taking: + // ``random(interval, interval * 1.5)``. + // + // .. attention:: + // + // Configuring ``rate_limited_retry_back_off`` will not by itself cause a request + // to be retried. You will still need to configure the right retry policy to match + // the responses from the upstream server. + message RateLimitedRetryBackOff { + // Specifies the reset headers (like ``Retry-After`` or ``X-RateLimit-Reset``) + // to match against the response. Headers are tried in order, and matched case + // insensitive. The first header to be parsed successfully is used. If no headers + // match the default exponential back-off is used instead. + repeated ResetHeader reset_headers = 1 [(validate.rules).repeated = {min_items: 1}]; + + // Specifies the maximum back off interval that Envoy will allow. If a reset + // header contains an interval longer than this then it will be discarded and + // the next header will be tried. Defaults to 300 seconds. + google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; + } + // Specifies the conditions under which retry takes place. These are the same // conditions documented for :ref:`config_http_filters_router_x-envoy-retry-on` and // :ref:`config_http_filters_router_x-envoy-retry-grpc-on`. @@ -1147,13 +1215,22 @@ message RetryPolicy { // HTTP status codes that should trigger a retry in addition to those specified by retry_on. repeated uint32 retriable_status_codes = 7; - // Specifies parameters that control retry back off. This parameter is optional, in which case the + // Specifies parameters that control exponential retry back off. This parameter is optional, in which case the // default base interval is 25 milliseconds or, if set, the current value of the // `upstream.base_retry_backoff_ms` runtime parameter. The default maximum interval is 10 times // the base interval. The documentation for :ref:`config_http_filters_router_x-envoy-max-retries` // describes Envoy's back-off algorithm. RetryBackOff retry_back_off = 8; + // Specifies parameters that control a retry back-off strategy that is used + // when the request is rate limited by the upstream server. The server may + // return a response header like ``Retry-After`` or ``X-RateLimit-Reset`` to + // provide feedback to the client on how long to wait before retrying. If + // configured, this back-off strategy will be used instead of the + // default exponential back off strategy (configured using `retry_back_off`) + // whenever a response includes the matching headers. + RateLimitedRetryBackOff rate_limited_retry_back_off = 11; + // HTTP response headers that trigger a retry if present in the response. A retry will be // triggered if any of the header matches match the upstream response headers. // The field is only consulted if 'retriable-headers' retry policy is active. diff --git a/api/envoy/config/route/v4alpha/route_components.proto b/api/envoy/config/route/v4alpha/route_components.proto index d57cdef06def..3aac37fbab3d 100644 --- a/api/envoy/config/route/v4alpha/route_components.proto +++ b/api/envoy/config/route/v4alpha/route_components.proto @@ -1028,10 +1028,15 @@ message RouteAction { } // HTTP retry :ref:`architecture overview `. -// [#next-free-field: 11] +// [#next-free-field: 12] message RetryPolicy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RetryPolicy"; + enum ResetHeaderFormat { + SECONDS = 0; + UNIX_TIMESTAMP = 1; + } + message RetryPriority { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RetryPolicy.RetryPriority"; @@ -1082,6 +1087,75 @@ message RetryPolicy { google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; } + message ResetHeader { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.route.v3.RetryPolicy.ResetHeader"; + + string name = 1 + [(validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}]; + + ResetHeaderFormat format = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // A retry back-off strategy that applies when the upstream server rate limits + // the request. + // + // Given this configuration: + // + // .. code-block:: yaml + // + // rate_limited_retry_back_off: + // reset_headers: + // - name: Retry-After + // format: SECONDS + // - name: X-RateLimit-Reset + // format: UNIX_TIMESTAMP + // max_interval: "300s" + // + // The following algorithm will apply: + // + // 1. If the response contains the header ``Retry-After`` its value must be on + // the form ``120`` (an integer that represents the number of seconds to + // wait before retrying). If so, this value is used as the back-off interval. + // 2. Otherwise, if the response contains the header ``X-RateLimit-Reset`` its + // value must be on the form ``1595320702`` (an integer that represents the + // point in time at which to retry, as a Unix timestamp in seconds). If so, + // the current time is subtracted from this value and the result is used as + // the back-off interval. + // 3. Otherwise, Envoy will use the default + // :ref:`exponential back-off ` + // strategy. + // + // No matter which format is used, if the resulting back-off interval exceeds + // ``max_interval`` it is discarded and the next header in ``reset_headers`` + // is tried. If a request timeout is configured for the route it will further + // limit how long the request will be allowed to run. + // + // To prevent many clients retrying at the same point in time jitter is added + // to the back-off interval, so the resulting interval is decided by taking: + // ``random(interval, interval * 1.5)``. + // + // .. attention:: + // + // Configuring ``rate_limited_retry_back_off`` will not by itself cause a request + // to be retried. You will still need to configure the right retry policy to match + // the responses from the upstream server. + message RateLimitedRetryBackOff { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.route.v3.RetryPolicy.RateLimitedRetryBackOff"; + + // Specifies the reset headers (like ``Retry-After`` or ``X-RateLimit-Reset``) + // to match against the response. Headers are tried in order, and matched case + // insensitive. The first header to be parsed successfully is used. If no headers + // match the default exponential back-off is used instead. + repeated ResetHeader reset_headers = 1 [(validate.rules).repeated = {min_items: 1}]; + + // Specifies the maximum back off interval that Envoy will allow. If a reset + // header contains an interval longer than this then it will be discarded and + // the next header will be tried. Defaults to 300 seconds. + google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; + } + // Specifies the conditions under which retry takes place. These are the same // conditions documented for :ref:`config_http_filters_router_x-envoy-retry-on` and // :ref:`config_http_filters_router_x-envoy-retry-grpc-on`. @@ -1124,13 +1198,22 @@ message RetryPolicy { // HTTP status codes that should trigger a retry in addition to those specified by retry_on. repeated uint32 retriable_status_codes = 7; - // Specifies parameters that control retry back off. This parameter is optional, in which case the + // Specifies parameters that control exponential retry back off. This parameter is optional, in which case the // default base interval is 25 milliseconds or, if set, the current value of the // `upstream.base_retry_backoff_ms` runtime parameter. The default maximum interval is 10 times // the base interval. The documentation for :ref:`config_http_filters_router_x-envoy-max-retries` // describes Envoy's back-off algorithm. RetryBackOff retry_back_off = 8; + // Specifies parameters that control a retry back-off strategy that is used + // when the request is rate limited by the upstream server. The server may + // return a response header like ``Retry-After`` or ``X-RateLimit-Reset`` to + // provide feedback to the client on how long to wait before retrying. If + // configured, this back-off strategy will be used instead of the + // default exponential back off strategy (configured using `retry_back_off`) + // whenever a response includes the matching headers. + RateLimitedRetryBackOff rate_limited_retry_back_off = 11; + // HTTP response headers that trigger a retry if present in the response. A retry will be // triggered if any of the header matches match the upstream response headers. // The field is only consulted if 'retriable-headers' retry policy is active. diff --git a/docs/root/configuration/http/http_filters/router_filter.rst b/docs/root/configuration/http/http_filters/router_filter.rst index 4ca285e9eda7..0c5ecb353783 100644 --- a/docs/root/configuration/http/http_filters/router_filter.rst +++ b/docs/root/configuration/http/http_filters/router_filter.rst @@ -42,7 +42,7 @@ A few notes on how Envoy does retries: retries. Thus if the request timeout is set to 3s, and the first request attempt takes 2.7s, the retry (including back-off) has .3s to complete. This is by design to avoid an exponential retry/timeout explosion. -* Envoy uses a fully jittered exponential back-off algorithm for retries with a default base +* By default, Envoy uses a fully jittered exponential back-off algorithm for retries with a default base interval of 25ms. Given a base interval B and retry number N, the back-off for the retry is in the range :math:`\big[0, (2^N-1)B\big)`. For example, given the default interval, the first retry will be delayed randomly by 0-24ms, the 2nd by 0-74ms, the 3rd by 0-174ms, and so on. The @@ -51,6 +51,13 @@ A few notes on how Envoy does retries: upstream.base_retry_backoff_ms runtime parameter. The back-off intervals can also be modified by configuring the retry policy's :ref:`retry back-off `. +* Envoy can also be configured to use feedback from the upstream server to decide the interval between + retries. Response headers like ``Retry-After`` or ``X-RateLimit-Reset`` instruct the client how long + to wait before re-trying. The retry policy's + :ref:`rate limited retry back off ` + strategy can be configured to expect a particular header, and if that header is present in the response Envoy + will use its value to decide the back-off. If the header is not present, or if it cannot be parsed + successfully, Envoy will use the default exponential back-off algorithm instead. .. _config_http_filters_router_x-envoy-retry-on: diff --git a/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst b/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst index 5d956c28d2b3..e4ba160313e7 100644 --- a/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst +++ b/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst @@ -74,6 +74,8 @@ Every cluster has a statistics tree rooted at *cluster..* with the followi upstream_rq_rx_reset, Counter, Total requests that were reset remotely upstream_rq_tx_reset, Counter, Total requests that were reset locally upstream_rq_retry, Counter, Total request retries + upstream_rq_retry_backoff_exponential, Counter, Total retries using the exponential backoff strategy + upstream_rq_retry_backoff_ratelimited, Counter, Total retries using the ratelimited backoff strategy upstream_rq_retry_limit_exceeded, Counter, Total requests not retried due to exceeding :ref:`the configured number of maximum retries ` upstream_rq_retry_success, Counter, Total request retry successes upstream_rq_retry_overflow, Counter, Total requests not retried due to circuit breaking or exceeding the :ref:`retry budget ` diff --git a/docs/root/intro/arch_overview/http/http_routing.rst b/docs/root/intro/arch_overview/http/http_routing.rst index ff24b0fd512e..884963dfcb80 100644 --- a/docs/root/intro/arch_overview/http/http_routing.rst +++ b/docs/root/intro/arch_overview/http/http_routing.rst @@ -106,8 +106,9 @@ Envoy allows retries to be configured both in the :ref:`route configuration ` as well as for specific requests via :ref:`request headers `. The following configurations are possible: -* **Maximum number of retries**: Envoy will continue to retry any number of times. An exponential - backoff algorithm is used between each retry. Additionally, *all retries are contained within the +* **Maximum number of retries**: Envoy will continue to retry any number of times. The intervals between + retries are decided either by an exponential backoff algorithm (the default), or based on feedback + from the upstream server via headers (if present). Additionally, *all retries are contained within the overall request timeout*. This avoids long request times due to a large number of retries. * **Retry conditions**: Envoy can retry on different types of conditions depending on application requirements. For example, network failure, all 5xx response codes, idempotent 4xx response codes, diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 052c77e5bdd0..326d4d586f03 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -63,6 +63,7 @@ New Features * ratelimit: added :ref:`enable_x_ratelimit_headers ` option to enable `X-RateLimit-*` headers as defined in `draft RFC `_. * rbac filter: added a log action to the :ref:`RBAC filter ` which sets dynamic metadata to inform access loggers whether to log. * redis: added fault injection support :ref:`fault injection for redis proxy `, described further in :ref:`configuration documentation `. +* router: added a new :ref:`rate limited retry back off ` strategy that uses headers like `Retry-After` or `X-RateLimit-Reset` to decide the back off interval. * router: added new :ref:`envoy-ratelimited` retry policy, which allows retrying envoy's own rate limited responses. diff --git a/generated_api_shadow/envoy/config/route/v3/route_components.proto b/generated_api_shadow/envoy/config/route/v3/route_components.proto index 0b8fe7908603..1c7e26120697 100644 --- a/generated_api_shadow/envoy/config/route/v3/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v3/route_components.proto @@ -1061,10 +1061,15 @@ message RouteAction { } // HTTP retry :ref:`architecture overview `. -// [#next-free-field: 11] +// [#next-free-field: 12] message RetryPolicy { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RetryPolicy"; + enum ResetHeaderFormat { + SECONDS = 0; + UNIX_TIMESTAMP = 1; + } + message RetryPriority { option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.route.RetryPolicy.RetryPriority"; @@ -1111,6 +1116,69 @@ message RetryPolicy { google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; } + message ResetHeader { + string name = 1 + [(validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}]; + + ResetHeaderFormat format = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // A retry back-off strategy that applies when the upstream server rate limits + // the request. + // + // Given this configuration: + // + // .. code-block:: yaml + // + // rate_limited_retry_back_off: + // reset_headers: + // - name: Retry-After + // format: SECONDS + // - name: X-RateLimit-Reset + // format: UNIX_TIMESTAMP + // max_interval: "300s" + // + // The following algorithm will apply: + // + // 1. If the response contains the header ``Retry-After`` its value must be on + // the form ``120`` (an integer that represents the number of seconds to + // wait before retrying). If so, this value is used as the back-off interval. + // 2. Otherwise, if the response contains the header ``X-RateLimit-Reset`` its + // value must be on the form ``1595320702`` (an integer that represents the + // point in time at which to retry, as a Unix timestamp in seconds). If so, + // the current time is subtracted from this value and the result is used as + // the back-off interval. + // 3. Otherwise, Envoy will use the default + // :ref:`exponential back-off ` + // strategy. + // + // No matter which format is used, if the resulting back-off interval exceeds + // ``max_interval`` it is discarded and the next header in ``reset_headers`` + // is tried. If a request timeout is configured for the route it will further + // limit how long the request will be allowed to run. + // + // To prevent many clients retrying at the same point in time jitter is added + // to the back-off interval, so the resulting interval is decided by taking: + // ``random(interval, interval * 1.5)``. + // + // .. attention:: + // + // Configuring ``rate_limited_retry_back_off`` will not by itself cause a request + // to be retried. You will still need to configure the right retry policy to match + // the responses from the upstream server. + message RateLimitedRetryBackOff { + // Specifies the reset headers (like ``Retry-After`` or ``X-RateLimit-Reset``) + // to match against the response. Headers are tried in order, and matched case + // insensitive. The first header to be parsed successfully is used. If no headers + // match the default exponential back-off is used instead. + repeated ResetHeader reset_headers = 1 [(validate.rules).repeated = {min_items: 1}]; + + // Specifies the maximum back off interval that Envoy will allow. If a reset + // header contains an interval longer than this then it will be discarded and + // the next header will be tried. Defaults to 300 seconds. + google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; + } + // Specifies the conditions under which retry takes place. These are the same // conditions documented for :ref:`config_http_filters_router_x-envoy-retry-on` and // :ref:`config_http_filters_router_x-envoy-retry-grpc-on`. @@ -1154,13 +1222,22 @@ message RetryPolicy { // HTTP status codes that should trigger a retry in addition to those specified by retry_on. repeated uint32 retriable_status_codes = 7; - // Specifies parameters that control retry back off. This parameter is optional, in which case the + // Specifies parameters that control exponential retry back off. This parameter is optional, in which case the // default base interval is 25 milliseconds or, if set, the current value of the // `upstream.base_retry_backoff_ms` runtime parameter. The default maximum interval is 10 times // the base interval. The documentation for :ref:`config_http_filters_router_x-envoy-max-retries` // describes Envoy's back-off algorithm. RetryBackOff retry_back_off = 8; + // Specifies parameters that control a retry back-off strategy that is used + // when the request is rate limited by the upstream server. The server may + // return a response header like ``Retry-After`` or ``X-RateLimit-Reset`` to + // provide feedback to the client on how long to wait before retrying. If + // configured, this back-off strategy will be used instead of the + // default exponential back off strategy (configured using `retry_back_off`) + // whenever a response includes the matching headers. + RateLimitedRetryBackOff rate_limited_retry_back_off = 11; + // HTTP response headers that trigger a retry if present in the response. A retry will be // triggered if any of the header matches match the upstream response headers. // The field is only consulted if 'retriable-headers' retry policy is active. diff --git a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto index 0cad79e8aa77..beaafe707ddd 100644 --- a/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto +++ b/generated_api_shadow/envoy/config/route/v4alpha/route_components.proto @@ -1056,10 +1056,15 @@ message RouteAction { } // HTTP retry :ref:`architecture overview `. -// [#next-free-field: 11] +// [#next-free-field: 12] message RetryPolicy { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RetryPolicy"; + enum ResetHeaderFormat { + SECONDS = 0; + UNIX_TIMESTAMP = 1; + } + message RetryPriority { option (udpa.annotations.versioning).previous_message_type = "envoy.config.route.v3.RetryPolicy.RetryPriority"; @@ -1110,6 +1115,75 @@ message RetryPolicy { google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; } + message ResetHeader { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.route.v3.RetryPolicy.ResetHeader"; + + string name = 1 + [(validate.rules).string = {min_bytes: 1 well_known_regex: HTTP_HEADER_NAME strict: false}]; + + ResetHeaderFormat format = 2 [(validate.rules).enum = {defined_only: true}]; + } + + // A retry back-off strategy that applies when the upstream server rate limits + // the request. + // + // Given this configuration: + // + // .. code-block:: yaml + // + // rate_limited_retry_back_off: + // reset_headers: + // - name: Retry-After + // format: SECONDS + // - name: X-RateLimit-Reset + // format: UNIX_TIMESTAMP + // max_interval: "300s" + // + // The following algorithm will apply: + // + // 1. If the response contains the header ``Retry-After`` its value must be on + // the form ``120`` (an integer that represents the number of seconds to + // wait before retrying). If so, this value is used as the back-off interval. + // 2. Otherwise, if the response contains the header ``X-RateLimit-Reset`` its + // value must be on the form ``1595320702`` (an integer that represents the + // point in time at which to retry, as a Unix timestamp in seconds). If so, + // the current time is subtracted from this value and the result is used as + // the back-off interval. + // 3. Otherwise, Envoy will use the default + // :ref:`exponential back-off ` + // strategy. + // + // No matter which format is used, if the resulting back-off interval exceeds + // ``max_interval`` it is discarded and the next header in ``reset_headers`` + // is tried. If a request timeout is configured for the route it will further + // limit how long the request will be allowed to run. + // + // To prevent many clients retrying at the same point in time jitter is added + // to the back-off interval, so the resulting interval is decided by taking: + // ``random(interval, interval * 1.5)``. + // + // .. attention:: + // + // Configuring ``rate_limited_retry_back_off`` will not by itself cause a request + // to be retried. You will still need to configure the right retry policy to match + // the responses from the upstream server. + message RateLimitedRetryBackOff { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.route.v3.RetryPolicy.RateLimitedRetryBackOff"; + + // Specifies the reset headers (like ``Retry-After`` or ``X-RateLimit-Reset``) + // to match against the response. Headers are tried in order, and matched case + // insensitive. The first header to be parsed successfully is used. If no headers + // match the default exponential back-off is used instead. + repeated ResetHeader reset_headers = 1 [(validate.rules).repeated = {min_items: 1}]; + + // Specifies the maximum back off interval that Envoy will allow. If a reset + // header contains an interval longer than this then it will be discarded and + // the next header will be tried. Defaults to 300 seconds. + google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {}}]; + } + // Specifies the conditions under which retry takes place. These are the same // conditions documented for :ref:`config_http_filters_router_x-envoy-retry-on` and // :ref:`config_http_filters_router_x-envoy-retry-grpc-on`. @@ -1152,13 +1226,22 @@ message RetryPolicy { // HTTP status codes that should trigger a retry in addition to those specified by retry_on. repeated uint32 retriable_status_codes = 7; - // Specifies parameters that control retry back off. This parameter is optional, in which case the + // Specifies parameters that control exponential retry back off. This parameter is optional, in which case the // default base interval is 25 milliseconds or, if set, the current value of the // `upstream.base_retry_backoff_ms` runtime parameter. The default maximum interval is 10 times // the base interval. The documentation for :ref:`config_http_filters_router_x-envoy-max-retries` // describes Envoy's back-off algorithm. RetryBackOff retry_back_off = 8; + // Specifies parameters that control a retry back-off strategy that is used + // when the request is rate limited by the upstream server. The server may + // return a response header like ``Retry-After`` or ``X-RateLimit-Reset`` to + // provide feedback to the client on how long to wait before retrying. If + // configured, this back-off strategy will be used instead of the + // default exponential back off strategy (configured using `retry_back_off`) + // whenever a response includes the matching headers. + RateLimitedRetryBackOff rate_limited_retry_back_off = 11; + // HTTP response headers that trigger a retry if present in the response. A retry will be // triggered if any of the header matches match the upstream response headers. // The field is only consulted if 'retriable-headers' retry policy is active. diff --git a/include/envoy/router/BUILD b/include/envoy/router/BUILD index 85b6058ed878..f3e5c450d04e 100644 --- a/include/envoy/router/BUILD +++ b/include/envoy/router/BUILD @@ -58,6 +58,7 @@ envoy_cc_library( "//include/envoy/access_log:access_log_interface", "//include/envoy/common:conn_pool_interface", "//include/envoy/common:matchers_interface", + "//include/envoy/common:time_interface", "//include/envoy/config:typed_metadata_interface", "//include/envoy/http:codec_interface", "//include/envoy/http:codes_interface", diff --git a/include/envoy/router/router.h b/include/envoy/router/router.h index 7d37cf02cf32..271d838d7b4f 100644 --- a/include/envoy/router/router.h +++ b/include/envoy/router/router.h @@ -150,6 +150,23 @@ class CorsPolicy { virtual bool shadowEnabled() const PURE; }; +/** + * An interface to be implemented by rate limited reset header parsers. + */ +class ResetHeaderParser { +public: + virtual ~ResetHeaderParser() = default; + + /** + * Iterate over the headers, choose the first one that matches by name, and try to parse its + * value. + */ + virtual absl::optional + parseInterval(TimeSource& time_source, const Http::HeaderMap& headers) const PURE; +}; + +using ResetHeaderParserSharedPtr = std::shared_ptr; + /** * Route level retry policy. */ @@ -235,6 +252,18 @@ class RetryPolicy { * @return absl::optional maximum retry interval */ virtual absl::optional maxInterval() const PURE; + + /** + * @return std::vector& list of reset header + * parsers that will be used to extract a retry back-off interval from response headers. + */ + virtual const std::vector& resetHeaders() const PURE; + + /** + * @return std::chrono::milliseconds upper limit placed on a retry + * back-off interval parsed from response headers. + */ + virtual std::chrono::milliseconds resetMaxInterval() const PURE; }; /** @@ -294,6 +323,14 @@ class RetryState { */ virtual bool enabled() PURE; + /** + * Attempts to parse any matching rate limited reset headers (RFC 7231), either in the form of an + * interval directly, or in the form of a unix timestamp relative to the current system time. + * @return the interval if parsing was successful. + */ + virtual absl::optional + parseResetInterval(const Http::ResponseHeaderMap& response_headers) const PURE; + /** * Determine whether a request should be retried based on the response headers. * @param response_headers supplies the response headers. diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index ebdc1575eb8f..9330b0e8fdb5 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -577,6 +577,8 @@ class PrioritySet { COUNTER(upstream_rq_pending_total) \ COUNTER(upstream_rq_per_try_timeout) \ COUNTER(upstream_rq_retry) \ + COUNTER(upstream_rq_retry_backoff_exponential) \ + COUNTER(upstream_rq_retry_backoff_ratelimited) \ COUNTER(upstream_rq_retry_limit_exceeded) \ COUNTER(upstream_rq_retry_overflow) \ COUNTER(upstream_rq_retry_success) \ diff --git a/source/common/common/backoff_strategy.cc b/source/common/common/backoff_strategy.cc index c9b5b61b733b..4f08e4332fa6 100644 --- a/source/common/common/backoff_strategy.cc +++ b/source/common/common/backoff_strategy.cc @@ -2,15 +2,15 @@ namespace Envoy { -JitteredBackOffStrategy::JitteredBackOffStrategy(uint64_t base_interval, uint64_t max_interval, - Random::RandomGenerator& random) +JitteredExponentialBackOffStrategy::JitteredExponentialBackOffStrategy( + uint64_t base_interval, uint64_t max_interval, Random::RandomGenerator& random) : base_interval_(base_interval), max_interval_(max_interval), next_interval_(base_interval), random_(random) { ASSERT(base_interval_ > 0); ASSERT(base_interval_ <= max_interval_); } -uint64_t JitteredBackOffStrategy::nextBackOffMs() { +uint64_t JitteredExponentialBackOffStrategy::nextBackOffMs() { const uint64_t backoff = next_interval_; ASSERT(backoff > 0); // Set next_interval_ to max_interval_ if doubling the interval would exceed the max or overflow. @@ -22,7 +22,16 @@ uint64_t JitteredBackOffStrategy::nextBackOffMs() { return std::min(random_.random() % backoff, max_interval_); } -void JitteredBackOffStrategy::reset() { next_interval_ = base_interval_; } +void JitteredExponentialBackOffStrategy::reset() { next_interval_ = base_interval_; } + +JitteredLowerBoundBackOffStrategy::JitteredLowerBoundBackOffStrategy( + uint64_t min_interval, Random::RandomGenerator& random) + : min_interval_(min_interval), random_(random) {} + +uint64_t JitteredLowerBoundBackOffStrategy::nextBackOffMs() { + // random(min_interval_, 1.5 * min_interval_) + return (random_.random() % (min_interval_ >> 1)) + min_interval_; +} FixedBackOffStrategy::FixedBackOffStrategy(uint64_t interval_ms) : interval_ms_(interval_ms) { ASSERT(interval_ms_ > 0); diff --git a/source/common/common/backoff_strategy.h b/source/common/common/backoff_strategy.h index 2484f3e11b20..c1928632979a 100644 --- a/source/common/common/backoff_strategy.h +++ b/source/common/common/backoff_strategy.h @@ -13,7 +13,7 @@ namespace Envoy { /** * Implementation of BackOffStrategy that uses a fully jittered exponential backoff algorithm. */ -class JitteredBackOffStrategy : public BackOffStrategy { +class JitteredExponentialBackOffStrategy : public BackOffStrategy { public: /** @@ -23,8 +23,8 @@ class JitteredBackOffStrategy : public BackOffStrategy { * @param max_interval the cap on the next backoff value. * @param random the random generator. */ - JitteredBackOffStrategy(uint64_t base_interval, uint64_t max_interval, - Random::RandomGenerator& random); + JitteredExponentialBackOffStrategy(uint64_t base_interval, uint64_t max_interval, + Random::RandomGenerator& random); // BackOffStrategy methods uint64_t nextBackOffMs() override; @@ -37,6 +37,28 @@ class JitteredBackOffStrategy : public BackOffStrategy { Random::RandomGenerator& random_; }; +/** + * Implementation of BackOffStrategy that returns random values in the range + * [min_interval, 1.5 * min_interval). + */ +class JitteredLowerBoundBackOffStrategy : public BackOffStrategy { +public: + /** + * Constructs fully jittered backoff strategy. + * @param min_interval the lower bound on the next backoff value. + * @param random the random generator. + */ + JitteredLowerBoundBackOffStrategy(uint64_t min_interval, Random::RandomGenerator& random); + + // BackOffStrategy methods + uint64_t nextBackOffMs() override; + void reset() override {} + +private: + const uint64_t min_interval_; + Random::RandomGenerator& random_; +}; + /** * Implementation of BackOffStrategy that uses a fixed backoff. */ diff --git a/source/common/config/datasource.cc b/source/common/config/datasource.cc index d3e286d0b27a..776061a61be2 100644 --- a/source/common/config/datasource.cc +++ b/source/common/config/datasource.cc @@ -64,8 +64,8 @@ RemoteAsyncDataProvider::RemoteAsyncDataProvider( } } - backoff_strategy_ = - std::make_unique(base_interval_ms, max_interval_ms, random); + backoff_strategy_ = std::make_unique(base_interval_ms, + max_interval_ms, random); retry_timer_ = dispatcher.createTimer([this]() -> void { start(); }); manager.add(init_target_); diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index b922d8c09f0f..f6673402df4f 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -48,8 +48,8 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, // TODO(htuch): Make this configurable. static constexpr uint32_t RetryInitialDelayMs = 500; static constexpr uint32_t RetryMaxDelayMs = 30000; // Do not cross more than 30s - backoff_strategy_ = - std::make_unique(RetryInitialDelayMs, RetryMaxDelayMs, random_); + backoff_strategy_ = std::make_unique( + RetryInitialDelayMs, RetryMaxDelayMs, random_); } void establishNewStream() { diff --git a/source/common/config/utility.h b/source/common/config/utility.h index 0e5f42c6c847..61b629f927f1 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -431,7 +431,8 @@ class Utility { "dns_failure_refresh_rate must have max_interval greater than " "or equal to the base_interval"); } - return std::make_unique(base_interval_ms, max_interval_ms, random); + return std::make_unique(base_interval_ms, max_interval_ms, + random); } return std::make_unique(dns_refresh_rate_ms); } diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index a4e2e7c86b84..6342b0a3af1f 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -152,10 +152,17 @@ class AsyncStreamImpl : public AsyncClient::Stream, return absl::nullopt; } absl::optional maxInterval() const override { return absl::nullopt; } + const std::vector& resetHeaders() const override { + return reset_headers_; + } + std::chrono::milliseconds resetMaxInterval() const override { + return std::chrono::milliseconds(300000); + } const std::vector retriable_status_codes_{}; const std::vector retriable_headers_{}; const std::vector retriable_request_headers_{}; + const std::vector reset_headers_{}; }; struct NullConfig : public Router::Config { diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 5f9e2b8cbc96..a2825b775dda 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -37,6 +37,7 @@ envoy_cc_library( ":header_formatter_lib", ":header_parser_lib", ":metadatamatchcriteria_lib", + ":reset_header_parser_lib", ":retry_state_lib", ":router_ratelimit_lib", ":tls_context_match_criteria_lib", @@ -378,6 +379,20 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "reset_header_parser_lib", + srcs = ["reset_header_parser.cc"], + hdrs = ["reset_header_parser.h"], + deps = [ + "//include/envoy/common:time_interface", + "//include/envoy/http:header_map_interface", + "//include/envoy/router:router_interface", + "//source/common/http:headers_lib", + "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + ], +) + envoy_cc_library( name = "string_accessor_lib", hdrs = ["string_accessor_impl.h"], diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 0cf162262a2f..d63c1711ac8a 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -33,6 +33,7 @@ #include "common/http/utility.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" +#include "common/router/reset_header_parser.h" #include "common/router/retry_state_impl.h" #include "common/runtime/runtime_features.h" #include "common/tracing/http_tracer_impl.h" @@ -120,6 +121,21 @@ RetryPolicyImpl::RetryPolicyImpl(const envoy::config::route::v3::RetryPolicy& re } } } + + if (retry_policy.has_rate_limited_retry_back_off()) { + reset_headers_ = ResetHeaderParserImpl::buildResetHeaderParserVector( + retry_policy.rate_limited_retry_back_off().reset_headers()); + + absl::optional reset_max_interval = + PROTOBUF_GET_OPTIONAL_MS(retry_policy.rate_limited_retry_back_off(), max_interval); + if (reset_max_interval.has_value()) { + std::chrono::milliseconds max_interval = reset_max_interval.value(); + if (max_interval.count() < 1) { + max_interval = std::chrono::milliseconds(1); + } + reset_max_interval_ = max_interval; + } + } } std::vector RetryPolicyImpl::retryHostPredicates() const { diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 4f4c0c8a7e1d..192da070ce53 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -272,6 +272,10 @@ class RetryPolicyImpl : public RetryPolicy { } absl::optional baseInterval() const override { return base_interval_; } absl::optional maxInterval() const override { return max_interval_; } + const std::vector& resetHeaders() const override { + return reset_headers_; + } + std::chrono::milliseconds resetMaxInterval() const override { return reset_max_interval_; } private: std::chrono::milliseconds per_try_timeout_{0}; @@ -294,6 +298,8 @@ class RetryPolicyImpl : public RetryPolicy { std::vector retriable_request_headers_; absl::optional base_interval_; absl::optional max_interval_; + std::vector reset_headers_{}; + std::chrono::milliseconds reset_max_interval_{300000}; ProtobufMessage::ValidationVisitor* validation_visitor_{}; }; diff --git a/source/common/router/reset_header_parser.cc b/source/common/router/reset_header_parser.cc new file mode 100644 index 000000000000..f5d61e9ca714 --- /dev/null +++ b/source/common/router/reset_header_parser.cc @@ -0,0 +1,68 @@ +#include "common/router/reset_header_parser.h" + +#include + +#include "common/common/assert.h" + +#include "absl/strings/numbers.h" + +namespace Envoy { +namespace Router { + +ResetHeaderParserImpl::ResetHeaderParserImpl( + const envoy::config::route::v3::RetryPolicy::ResetHeader& config) + : name_(config.name()) { + switch (config.format()) { + case envoy::config::route::v3::RetryPolicy::SECONDS: + format_ = ResetHeaderFormat::Seconds; + break; + case envoy::config::route::v3::RetryPolicy::UNIX_TIMESTAMP: + format_ = ResetHeaderFormat::UnixTimestamp; + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +absl::optional +ResetHeaderParserImpl::parseInterval(TimeSource& time_source, + const Http::HeaderMap& headers) const { + const Http::HeaderEntry* header = headers.get(name_); + + if (header == nullptr) { + return absl::nullopt; + } + + const auto& header_value = header->value().getStringView(); + uint64_t num_seconds{}; + + switch (format_) { + case ResetHeaderFormat::Seconds: + if (absl::SimpleAtoi(header_value, &num_seconds)) { + return absl::optional(num_seconds * 1000UL); + } + break; + + case ResetHeaderFormat::UnixTimestamp: + if (absl::SimpleAtoi(header_value, &num_seconds)) { + const auto time_now = time_source.systemTime().time_since_epoch(); + const uint64_t timestamp = std::chrono::duration_cast(time_now).count(); + + if (num_seconds < timestamp) { + return absl::nullopt; + } + + const uint64_t interval = num_seconds - timestamp; + return absl::optional(interval * 1000UL); + } + break; + + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + + return absl::nullopt; +} + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/reset_header_parser.h b/source/common/router/reset_header_parser.h new file mode 100644 index 000000000000..776486d339b1 --- /dev/null +++ b/source/common/router/reset_header_parser.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/time.h" +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/http/header_map.h" +#include "envoy/router/router.h" + +#include "common/protobuf/protobuf.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Router { + +enum class ResetHeaderFormat { Seconds, UnixTimestamp }; + +/** + * A ResetHeaderParser specifies a header name and a format to match against + * response headers that are used to signal a rate limit interval reset, such + * as Retry-After or X-RateLimit-Reset. + */ +class ResetHeaderParserImpl : public ResetHeaderParser { +public: + /** + * Build a vector of ResetHeaderParserSharedPtr given input config. + */ + static std::vector buildResetHeaderParserVector( + const Protobuf::RepeatedPtrField& + reset_headers) { + std::vector ret; + for (const auto& reset_header : reset_headers) { + ret.emplace_back(std::make_shared(reset_header)); + } + return ret; + } + + ResetHeaderParserImpl(const envoy::config::route::v3::RetryPolicy::ResetHeader& config); + + const Http::LowerCaseString& name() const { return name_; } + ResetHeaderFormat format() const { return format_; } + + /** + * Iterate over the headers, choose the first one that matches by name, and try to parse its + * value. + */ + absl::optional + parseInterval(TimeSource& time_source, const Http::HeaderMap& headers) const override; + +private: + const Http::LowerCaseString name_; + ResetHeaderFormat format_; +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/retry_state_impl.cc b/source/common/router/retry_state_impl.cc index 9912f041709e..938850c47f42 100644 --- a/source/common/router/retry_state_impl.cc +++ b/source/common/router/retry_state_impl.cc @@ -38,14 +38,14 @@ RetryStatePtr RetryStateImpl::create(const RetryPolicy& route_policy, const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, Runtime::Loader& runtime, Random::RandomGenerator& random, Event::Dispatcher& dispatcher, - Upstream::ResourcePriority priority) { + TimeSource& time_source, Upstream::ResourcePriority priority) { RetryStatePtr ret; // We short circuit here and do not bother with an allocation if there is no chance we will retry. if (request_headers.EnvoyRetryOn() || request_headers.EnvoyRetryGrpcOn() || route_policy.retryOn()) { ret.reset(new RetryStateImpl(route_policy, request_headers, cluster, vcluster, runtime, random, - dispatcher, priority)); + dispatcher, time_source, priority)); } // Consume all retry related headers to avoid them being propagated to the upstream @@ -66,14 +66,17 @@ RetryStateImpl::RetryStateImpl(const RetryPolicy& route_policy, Http::RequestHeaderMap& request_headers, const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, Runtime::Loader& runtime, Random::RandomGenerator& random, - Event::Dispatcher& dispatcher, Upstream::ResourcePriority priority) + Event::Dispatcher& dispatcher, TimeSource& time_source, + Upstream::ResourcePriority priority) : cluster_(cluster), vcluster_(vcluster), runtime_(runtime), random_(random), - dispatcher_(dispatcher), retry_on_(route_policy.retryOn()), + dispatcher_(dispatcher), time_source_(time_source), retry_on_(route_policy.retryOn()), retries_remaining_(route_policy.numRetries()), priority_(priority), retry_host_predicates_(route_policy.retryHostPredicates()), retry_priority_(route_policy.retryPriority()), retriable_status_codes_(route_policy.retriableStatusCodes()), - retriable_headers_(route_policy.retriableHeaders()) { + retriable_headers_(route_policy.retriableHeaders()), + reset_headers_(route_policy.resetHeaders()), + reset_max_interval_(route_policy.resetMaxInterval()) { std::chrono::milliseconds base_interval( runtime_.snapshot().getInteger("upstream.base_retry_backoff_ms", 25)); @@ -88,8 +91,8 @@ RetryStateImpl::RetryStateImpl(const RetryPolicy& route_policy, max_interval = *route_policy.maxInterval(); } - backoff_strategy_ = std::make_unique(base_interval.count(), - max_interval.count(), random_); + backoff_strategy_ = std::make_unique( + base_interval.count(), max_interval.count(), random_); host_selection_max_attempts_ = route_policy.hostSelectionMaxAttempts(); // Merge in the headers. @@ -156,8 +159,23 @@ void RetryStateImpl::enableBackoffTimer() { retry_timer_ = dispatcher_.createTimer([this]() -> void { callback_(); }); } - // We use a fully jittered exponential backoff algorithm. - retry_timer_->enableTimer(std::chrono::milliseconds(backoff_strategy_->nextBackOffMs())); + if (ratelimited_backoff_strategy_ != nullptr) { + // If we have a backoff strategy based on rate limit feedback from the response we use it. + retry_timer_->enableTimer( + std::chrono::milliseconds(ratelimited_backoff_strategy_->nextBackOffMs())); + + // The strategy is only valid for the response that sent the ratelimit reset header and cannot + // be reused. + ratelimited_backoff_strategy_.reset(); + + cluster_.stats().upstream_rq_retry_backoff_ratelimited_.inc(); + + } else { + // Otherwise we use a fully jittered exponential backoff algorithm. + retry_timer_->enableTimer(std::chrono::milliseconds(backoff_strategy_->nextBackOffMs())); + + cluster_.stats().upstream_rq_retry_backoff_exponential_.inc(); + } } std::pair RetryStateImpl::parseRetryOn(absl::string_view config) { @@ -212,6 +230,18 @@ std::pair RetryStateImpl::parseRetryGrpcOn(absl::string_view ret return {ret, all_fields_valid}; } +absl::optional +RetryStateImpl::parseResetInterval(const Http::ResponseHeaderMap& response_headers) const { + for (const auto& reset_header : reset_headers_) { + const auto interval = reset_header->parseInterval(time_source_, response_headers); + if (interval.has_value() && interval.value() <= reset_max_interval_) { + return interval; + } + } + + return absl::nullopt; +} + void RetryStateImpl::resetRetry() { if (callback_) { cluster_.resourceManager(priority_).retries().dec(); @@ -273,7 +303,19 @@ RetryStatus RetryStateImpl::shouldRetry(bool would_retry, DoRetryCallback callba RetryStatus RetryStateImpl::shouldRetryHeaders(const Http::ResponseHeaderMap& response_headers, DoRetryCallback callback) { - return shouldRetry(wouldRetryFromHeaders(response_headers), callback); + const bool would_retry = wouldRetryFromHeaders(response_headers); + + // Yes, we will retry based on the headers - try to parse a rate limited reset interval from the + // response. + if (would_retry && !reset_headers_.empty()) { + const auto backoff_interval = parseResetInterval(response_headers); + if (backoff_interval.has_value()) { + ratelimited_backoff_strategy_ = std::make_unique( + backoff_interval.value().count(), random_); + } + } + + return shouldRetry(would_retry, callback); } RetryStatus RetryStateImpl::shouldRetryReset(Http::StreamResetReason reset_reason, diff --git a/source/common/router/retry_state_impl.h b/source/common/router/retry_state_impl.h index 9b0a19a77911..8931b5bf3751 100644 --- a/source/common/router/retry_state_impl.h +++ b/source/common/router/retry_state_impl.h @@ -29,7 +29,8 @@ class RetryStateImpl : public RetryState { Http::RequestHeaderMap& request_headers, const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, Runtime::Loader& runtime, Random::RandomGenerator& random, - Event::Dispatcher& dispatcher, Upstream::ResourcePriority priority); + Event::Dispatcher& dispatcher, TimeSource& time_source, + Upstream::ResourcePriority priority); ~RetryStateImpl() override; /** @@ -52,6 +53,8 @@ class RetryStateImpl : public RetryState { // Router::RetryState bool enabled() override { return retry_on_ != 0; } + absl::optional + parseResetInterval(const Http::ResponseHeaderMap& response_headers) const override; RetryStatus shouldRetryHeaders(const Http::ResponseHeaderMap& response_headers, DoRetryCallback callback) override; // Returns true if the retry policy would retry the passed headers. Does not @@ -92,7 +95,8 @@ class RetryStateImpl : public RetryState { RetryStateImpl(const RetryPolicy& route_policy, Http::RequestHeaderMap& request_headers, const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, Runtime::Loader& runtime, Random::RandomGenerator& random, - Event::Dispatcher& dispatcher, Upstream::ResourcePriority priority); + Event::Dispatcher& dispatcher, TimeSource& time_source, + Upstream::ResourcePriority priority); void enableBackoffTimer(); void resetRetry(); @@ -104,17 +108,21 @@ class RetryStateImpl : public RetryState { Runtime::Loader& runtime_; Random::RandomGenerator& random_; Event::Dispatcher& dispatcher_; + TimeSource& time_source_; uint32_t retry_on_{}; uint32_t retries_remaining_{}; DoRetryCallback callback_; Event::TimerPtr retry_timer_; Upstream::ResourcePriority priority_; BackOffStrategyPtr backoff_strategy_; + BackOffStrategyPtr ratelimited_backoff_strategy_{}; std::vector retry_host_predicates_; Upstream::RetryPrioritySharedPtr retry_priority_; uint32_t host_selection_max_attempts_; std::vector retriable_status_codes_; std::vector retriable_headers_; + std::vector reset_headers_{}; + std::chrono::milliseconds reset_max_interval_{}; }; } // namespace Router diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 88db133d6e2c..9e2e00128a4a 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -569,9 +569,9 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::RequestHeaderMap& headers, // Ensure an http transport scheme is selected before continuing with decoding. ASSERT(headers.Scheme()); - retry_state_ = createRetryState(route_entry_->retryPolicy(), headers, *cluster_, - request_vcluster_, config_.runtime_, config_.random_, - callbacks_->dispatcher(), route_entry_->priority()); + retry_state_ = createRetryState( + route_entry_->retryPolicy(), headers, *cluster_, request_vcluster_, config_.runtime_, + config_.random_, callbacks_->dispatcher(), config_.timeSource(), route_entry_->priority()); // Determine which shadow policies to use. It's possible that we don't do any shadowing due to // runtime keys. @@ -1575,13 +1575,15 @@ uint32_t Filter::numRequestsAwaitingHeaders() { [](const auto& req) -> bool { return req->awaitingHeaders(); }); } -RetryStatePtr -ProdFilter::createRetryState(const RetryPolicy& policy, Http::RequestHeaderMap& request_headers, - const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, - Runtime::Loader& runtime, Random::RandomGenerator& random, - Event::Dispatcher& dispatcher, Upstream::ResourcePriority priority) { +RetryStatePtr ProdFilter::createRetryState(const RetryPolicy& policy, + Http::RequestHeaderMap& request_headers, + const Upstream::ClusterInfo& cluster, + const VirtualCluster* vcluster, Runtime::Loader& runtime, + Random::RandomGenerator& random, + Event::Dispatcher& dispatcher, TimeSource& time_source, + Upstream::ResourcePriority priority) { return RetryStateImpl::create(policy, request_headers, cluster, vcluster, runtime, random, - dispatcher, priority); + dispatcher, time_source, priority); } } // namespace Router diff --git a/source/common/router/router.h b/source/common/router/router.h index 65ef129ccf3a..e1be0571e2b6 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -473,11 +473,13 @@ class Filter : Logger::Loggable, bool dropped); void chargeUpstreamAbort(Http::Code code, bool dropped, UpstreamRequest& upstream_request); void cleanup(); - virtual RetryStatePtr - createRetryState(const RetryPolicy& policy, Http::RequestHeaderMap& request_headers, - const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, - Runtime::Loader& runtime, Random::RandomGenerator& random, - Event::Dispatcher& dispatcher, Upstream::ResourcePriority priority) PURE; + virtual RetryStatePtr createRetryState(const RetryPolicy& policy, + Http::RequestHeaderMap& request_headers, + const Upstream::ClusterInfo& cluster, + const VirtualCluster* vcluster, Runtime::Loader& runtime, + Random::RandomGenerator& random, + Event::Dispatcher& dispatcher, TimeSource& time_source, + Upstream::ResourcePriority priority) PURE; std::unique_ptr createConnPool(); UpstreamRequestPtr createUpstreamRequest(); @@ -567,6 +569,7 @@ class ProdFilter : public Filter { const Upstream::ClusterInfo& cluster, const VirtualCluster* vcluster, Runtime::Loader& runtime, Random::RandomGenerator& random, Event::Dispatcher& dispatcher, + TimeSource& time_source, Upstream::ResourcePriority priority) override; }; diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index 21fa74e34588..4d66e0f6c63f 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -50,8 +50,8 @@ HdsDelegate::HdsDelegate(Stats::Scope& scope, Grpc::RawAsyncClientPtr async_clie api_(api) { health_check_request_.mutable_health_check_request()->mutable_node()->MergeFrom( local_info_.node()); - backoff_strategy_ = std::make_unique(RetryInitialDelayMilliseconds, - RetryMaxDelayMilliseconds, random_); + backoff_strategy_ = std::make_unique( + RetryInitialDelayMilliseconds, RetryMaxDelayMilliseconds, random_); hds_retry_timer_ = dispatcher.createTimer([this]() -> void { establishNewStream(); }); hds_stream_response_timer_ = dispatcher.createTimer([this]() -> void { sendResponse(); }); diff --git a/test/common/common/backoff_strategy_test.cc b/test/common/common/backoff_strategy_test.cc index 20ffe937065c..bfc540234c7b 100644 --- a/test/common/common/backoff_strategy_test.cc +++ b/test/common/common/backoff_strategy_test.cc @@ -9,20 +9,20 @@ using testing::Return; namespace Envoy { -TEST(BackOffStrategyTest, JitteredBackOffBasicFlow) { +TEST(ExponentialBackOffStrategyTest, JitteredBackOffBasicFlow) { NiceMock random; ON_CALL(random, random()).WillByDefault(Return(27)); - JitteredBackOffStrategy jittered_back_off(25, 30, random); + JitteredExponentialBackOffStrategy jittered_back_off(25, 30, random); EXPECT_EQ(2, jittered_back_off.nextBackOffMs()); EXPECT_EQ(27, jittered_back_off.nextBackOffMs()); } -TEST(BackOffStrategyTest, JitteredBackOffBasicReset) { +TEST(ExponentialBackOffStrategyTest, JitteredBackOffBasicReset) { NiceMock random; ON_CALL(random, random()).WillByDefault(Return(27)); - JitteredBackOffStrategy jittered_back_off(25, 30, random); + JitteredExponentialBackOffStrategy jittered_back_off(25, 30, random); EXPECT_EQ(2, jittered_back_off.nextBackOffMs()); EXPECT_EQ(27, jittered_back_off.nextBackOffMs()); @@ -30,22 +30,23 @@ TEST(BackOffStrategyTest, JitteredBackOffBasicReset) { EXPECT_EQ(2, jittered_back_off.nextBackOffMs()); // Should start from start } -TEST(BackOffStrategyTest, JitteredBackOffDoesntOverflow) { +TEST(ExponentialBackOffStrategyTest, JitteredBackOffDoesntOverflow) { NiceMock random; ON_CALL(random, random()).WillByDefault(Return(std::numeric_limits::max() - 1)); - JitteredBackOffStrategy jittered_back_off(1, std::numeric_limits::max(), random); + JitteredExponentialBackOffStrategy jittered_back_off(1, std::numeric_limits::max(), + random); for (int iter = 0; iter < 100; ++iter) { EXPECT_GE(std::numeric_limits::max(), jittered_back_off.nextBackOffMs()); } EXPECT_EQ(std::numeric_limits::max() - 1, jittered_back_off.nextBackOffMs()); } -TEST(BackOffStrategyTest, JitteredBackOffWithMaxInterval) { +TEST(ExponentialBackOffStrategyTest, JitteredBackOffWithMaxInterval) { NiceMock random; ON_CALL(random, random()).WillByDefault(Return(9999)); - JitteredBackOffStrategy jittered_back_off(5, 100, random); + JitteredExponentialBackOffStrategy jittered_back_off(5, 100, random); EXPECT_EQ(4, jittered_back_off.nextBackOffMs()); EXPECT_EQ(9, jittered_back_off.nextBackOffMs()); EXPECT_EQ(19, jittered_back_off.nextBackOffMs()); @@ -55,11 +56,11 @@ TEST(BackOffStrategyTest, JitteredBackOffWithMaxInterval) { EXPECT_EQ(99, jittered_back_off.nextBackOffMs()); } -TEST(BackOffStrategyTest, JitteredBackOffWithMaxIntervalReset) { +TEST(ExponentialBackOffStrategyTest, JitteredBackOffWithMaxIntervalReset) { NiceMock random; ON_CALL(random, random()).WillByDefault(Return(9999)); - JitteredBackOffStrategy jittered_back_off(5, 100, random); + JitteredExponentialBackOffStrategy jittered_back_off(5, 100, random); EXPECT_EQ(4, jittered_back_off.nextBackOffMs()); EXPECT_EQ(9, jittered_back_off.nextBackOffMs()); EXPECT_EQ(19, jittered_back_off.nextBackOffMs()); @@ -78,7 +79,23 @@ TEST(BackOffStrategyTest, JitteredBackOffWithMaxIntervalReset) { EXPECT_EQ(99, jittered_back_off.nextBackOffMs()); } -TEST(BackOffStrategyTest, FixedBackOffBasicReset) { +TEST(LowerBoundBackOffStrategyTest, JitteredBackOffWithLowRandomValue) { + NiceMock random; + ON_CALL(random, random()).WillByDefault(Return(22)); + + JitteredLowerBoundBackOffStrategy jittered_lower_bound_back_off(500, random); + EXPECT_EQ(522, jittered_lower_bound_back_off.nextBackOffMs()); +} + +TEST(LowerBoundBackOffStrategyTest, JitteredBackOffWithHighRandomValue) { + NiceMock random; + ON_CALL(random, random()).WillByDefault(Return(9999)); + + JitteredLowerBoundBackOffStrategy jittered_lower_bound_back_off(500, random); + EXPECT_EQ(749, jittered_lower_bound_back_off.nextBackOffMs()); +} + +TEST(FixedBackOffStrategyTest, FixedBackOffBasicReset) { FixedBackOffStrategy fixed_back_off(30); EXPECT_EQ(30, fixed_back_off.nextBackOffMs()); EXPECT_EQ(30, fixed_back_off.nextBackOffMs()); diff --git a/test/common/config/utility_test.cc b/test/common/config/utility_test.cc index 23ab3e0b0235..d755486a872b 100644 --- a/test/common/config/utility_test.cc +++ b/test/common/config/utility_test.cc @@ -265,7 +265,7 @@ TEST(UtilityTest, PrepareDnsRefreshStrategy) { BackOffStrategyPtr strategy = Utility::prepareDnsRefreshStrategy(cluster, 5000, random); - EXPECT_NE(nullptr, dynamic_cast(strategy.get())); + EXPECT_NE(nullptr, dynamic_cast(strategy.get())); } { diff --git a/test/common/router/BUILD b/test/common/router/BUILD index a377e5672fdd..0fad319d411c 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -61,6 +61,18 @@ envoy_cc_fuzz_test( ], ) +envoy_cc_test( + name = "reset_header_parser_test", + srcs = ["reset_header_parser_test.cc"], + deps = [ + "//source/common/http:header_utility_lib", + "//source/common/router:reset_header_parser_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "rds_impl_test", srcs = ["rds_impl_test.cc"], @@ -155,6 +167,7 @@ envoy_cc_test( srcs = ["retry_state_impl_test.cc"], deps = [ "//source/common/http:header_map_lib", + "//source/common/router:reset_header_parser_lib", "//source/common/router:retry_state_lib", "//source/common/upstream:resource_manager_lib", "//test/mocks/router:router_mocks", diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 7585b4ec0524..97bd2fd21528 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -306,6 +306,7 @@ most_specific_header_mutations_wins: {0} Stats::TestSymbolTable symbol_table_; Api::ApiPtr api_; NiceMock factory_context_; + Event::SimulatedTimeSystem test_time_; }; class RouteMatcherTest : public testing::Test, public ConfigImplTestBase {}; @@ -3580,6 +3581,86 @@ TEST_F(RouteMatcherTest, InvalidRetryBackOff) { "retry_policy.max_interval must greater than or equal to the base_interval"); } +TEST_F(RouteMatcherTest, RateLimitedRetryBackOff) { + const std::string yaml = R"EOF( +virtual_hosts: +- name: www + domains: + - www.lyft.com + routes: + - match: + prefix: "/no-backoff" + route: + cluster: www + - match: + prefix: "/sub-ms-interval" + route: + cluster: www + retry_policy: + rate_limited_retry_back_off: + reset_headers: + - name: Retry-After + format: SECONDS + max_interval: 0.0001s # < 1 ms + - match: + prefix: "/typical-backoff" + route: + cluster: www + retry_policy: + rate_limited_retry_back_off: + reset_headers: + - name: Retry-After + format: SECONDS + - name: RateLimit-Reset + format: UNIX_TIMESTAMP + max_interval: 0.050s + )EOF"; + + const time_t known_date_time = 1000000000; + test_time_.setSystemTime(std::chrono::system_clock::from_time_t(known_date_time)); + + TestConfigImpl config(parseRouteConfigurationFromYaml(yaml), factory_context_, true); + + // has no ratelimit retry back off + EXPECT_EQ(true, config.route(genHeaders("www.lyft.com", "/no-backoff", "GET"), 0) + ->routeEntry() + ->retryPolicy() + .resetHeaders() + .empty()); + EXPECT_EQ(std::chrono::milliseconds(300000), + config.route(genHeaders("www.lyft.com", "/no-backoff", "GET"), 0) + ->routeEntry() + ->retryPolicy() + .resetMaxInterval()); + + // has sub millisecond interval + EXPECT_EQ(1, config.route(genHeaders("www.lyft.com", "/sub-ms-interval", "GET"), 0) + ->routeEntry() + ->retryPolicy() + .resetHeaders() + .size()); + EXPECT_EQ(std::chrono::milliseconds(1), + config.route(genHeaders("www.lyft.com", "/sub-ms-interval", "GET"), 0) + ->routeEntry() + ->retryPolicy() + .resetMaxInterval()); + + // a typical configuration + Http::TestRequestHeaderMapImpl headers = genHeaders("www.lyft.com", "/typical-backoff", "GET"); + const auto& retry_policy = config.route(headers, 0)->routeEntry()->retryPolicy(); + EXPECT_EQ(2, retry_policy.resetHeaders().size()); + + Http::TestResponseHeaderMapImpl expected_0{{"Retry-After", "2"}}; + Http::TestResponseHeaderMapImpl expected_1{{"RateLimit-Reset", "1000000005"}}; + + EXPECT_EQ(std::chrono::milliseconds(2000), + retry_policy.resetHeaders()[0]->parseInterval(test_time_.timeSystem(), expected_0)); + EXPECT_EQ(std::chrono::milliseconds(5000), + retry_policy.resetHeaders()[1]->parseInterval(test_time_.timeSystem(), expected_1)); + + EXPECT_EQ(std::chrono::milliseconds(50), retry_policy.resetMaxInterval()); +} + TEST_F(RouteMatcherTest, HedgeRouteLevel) { const std::string yaml = R"EOF( virtual_hosts: diff --git a/test/common/router/reset_header_parser_test.cc b/test/common/router/reset_header_parser_test.cc new file mode 100644 index 000000000000..bb6135390db4 --- /dev/null +++ b/test/common/router/reset_header_parser_test.cc @@ -0,0 +1,210 @@ +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/http/protocol.h" +#include "envoy/json/json_object.h" + +#include "common/json/json_loader.h" +#include "common/router/reset_header_parser.h" + +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Router { +namespace { + +envoy::config::route::v3::RetryPolicy::ResetHeader +parseResetHeaderParserFromYaml(const std::string& yaml) { + envoy::config::route::v3::RetryPolicy::ResetHeader reset_header; + TestUtility::loadFromYaml(yaml, reset_header); + return reset_header; +} + +TEST(ResetHeaderParserConstructorTest, FormatUnset) { + const std::string yaml = R"EOF( +name: retry-after + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + EXPECT_EQ("retry-after", reset_header_parser.name().get()); + EXPECT_EQ(ResetHeaderFormat::Seconds, reset_header_parser.format()); +} + +TEST(ResetHeaderParserConstructorTest, FormatSeconds) { + const std::string yaml = R"EOF( +name: retry-after +format: SECONDS + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + EXPECT_EQ("retry-after", reset_header_parser.name().get()); + EXPECT_EQ(ResetHeaderFormat::Seconds, reset_header_parser.format()); +} + +TEST(ResetHeaderParserConstructorTest, FormatUnixTimestamp) { + const std::string yaml = R"EOF( +name: retry-after +format: UNIX_TIMESTAMP + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + EXPECT_EQ("retry-after", reset_header_parser.name().get()); + EXPECT_EQ(ResetHeaderFormat::UnixTimestamp, reset_header_parser.format()); +} + +class ResetHeaderParserParseIntervalTest : public testing::Test { +public: + ResetHeaderParserParseIntervalTest() { + const time_t known_date_time = 1000000000; + test_time_.setSystemTime(std::chrono::system_clock::from_time_t(known_date_time)); + } + + Event::SimulatedTimeSystem test_time_; +}; + +TEST_F(ResetHeaderParserParseIntervalTest, NoHeaderMatches) { + const std::string yaml = R"EOF( +name: retry-after +format: SECONDS + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "200"}}; + + EXPECT_EQ(absl::nullopt, + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesButUnsupportedFormatDate) { + const std::string yaml = R"EOF( +name: retry-after +format: SECONDS + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{ + {"retry-after", "Fri, 17 Jul 2020 11:59:51 GMT"}}; + + EXPECT_EQ(absl::nullopt, + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesButUnsupportedFormatFloat) { + const std::string yaml = R"EOF( +name: retry-after +format: SECONDS + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{"retry-after", "2.5"}}; + + EXPECT_EQ(absl::nullopt, + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesSupportedFormatSeconds) { + const std::string yaml = R"EOF( +name: retry-after +format: SECONDS + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{"retry-after", "5"}}; + + EXPECT_EQ(absl::optional(5000), + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesSupportedFormatSecondsCaseInsensitive) { + const std::string yaml = R"EOF( +name: retry-after +format: SECONDS + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{"Retry-After", "5"}}; + + EXPECT_EQ(absl::optional(5000), + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesButUnsupportedFormatTimestampFloat) { + const std::string yaml = R"EOF( +name: retry-after +format: UNIX_TIMESTAMP + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{"retry-after", "1595320702.1234"}}; + + EXPECT_EQ(absl::nullopt, + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesSupportedFormatTimestampButInThePast) { + const std::string yaml = R"EOF( +name: retry-after +format: UNIX_TIMESTAMP + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{"retry-after", "999999999"}}; + + EXPECT_EQ(absl::nullopt, + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesSupportedFormatTimestampEmptyInterval) { + const std::string yaml = R"EOF( +name: retry-after +format: UNIX_TIMESTAMP + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{"retry-after", "1000000000"}}; + + EXPECT_EQ(absl::optional(0), + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +TEST_F(ResetHeaderParserParseIntervalTest, HeaderMatchesSupportedFormatTimestampNonEmptyInterval) { + const std::string yaml = R"EOF( +name: retry-after +format: UNIX_TIMESTAMP + )EOF"; + + ResetHeaderParserImpl reset_header_parser = + ResetHeaderParserImpl(parseResetHeaderParserFromYaml(yaml)); + + Http::TestResponseHeaderMapImpl response_headers{{"retry-after", "1000000007"}}; + + EXPECT_EQ(absl::optional(7000), + reset_header_parser.parseInterval(test_time_.timeSystem(), response_headers)); +} + +} // namespace +} // namespace Router +} // namespace Envoy diff --git a/test/common/router/retry_state_impl_test.cc b/test/common/router/retry_state_impl_test.cc index 6f3d6441baaf..f7bc5cb87c36 100644 --- a/test/common/router/retry_state_impl_test.cc +++ b/test/common/router/retry_state_impl_test.cc @@ -4,6 +4,7 @@ #include "envoy/stats/stats.h" #include "common/http/header_map_impl.h" +#include "common/router/reset_header_parser.h" #include "common/router/retry_state_impl.h" #include "common/upstream/resource_manager_impl.h" @@ -42,7 +43,8 @@ class RouterRetryStateImplTest : public testing::Test { void setup(Http::RequestHeaderMap& request_headers) { state_ = RetryStateImpl::create(policy_, request_headers, cluster_, &virtual_cluster_, runtime_, - random_, dispatcher_, Upstream::ResourcePriority::Default); + random_, dispatcher_, test_time_.timeSystem(), + Upstream::ResourcePriority::Default); } void expectTimerCreateAndEnable() { @@ -123,6 +125,7 @@ class RouterRetryStateImplTest : public testing::Test { void TearDown() override { cleanupOutstandingResources(); } + Event::SimulatedTimeSystem test_time_; NiceMock policy_; NiceMock cluster_; TestVirtualCluster virtual_cluster_; @@ -925,6 +928,128 @@ TEST_F(RouterRetryStateImplTest, CustomBackOffIntervalDefaultMax) { retry_timer_->invokeCallback(); } +TEST_F(RouterRetryStateImplTest, ParseRateLimitedResetInterval) { + // Set a fixed system time to be used for all these tests + const time_t known_date_time = 1000000000; + test_time_.setSystemTime(std::chrono::system_clock::from_time_t(known_date_time)); + + Protobuf::RepeatedPtrField reset_headers; + auto* reset_header_1 = reset_headers.Add(); + reset_header_1->set_name("Retry-After"); + reset_header_1->set_format(envoy::config::route::v3::RetryPolicy::SECONDS); + + auto* reset_header_2 = reset_headers.Add(); + reset_header_2->set_name("X-RateLimit-Reset"); + reset_header_2->set_format(envoy::config::route::v3::RetryPolicy::UNIX_TIMESTAMP); + + policy_.reset_headers_ = ResetHeaderParserImpl::buildResetHeaderParserVector(reset_headers); + + // Failure case: Matches reset header (seconds) but exceeds max_interval (>5min) + { + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-retry-on", "5xx"}}; + setup(request_headers); + EXPECT_TRUE(state_->enabled()); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "429"}, {"Retry-After", "301"}}; + EXPECT_EQ(absl::nullopt, state_->parseResetInterval(response_headers)); + } + + // Failure case: Matches reset header (timestamp) but exceeds max_interval (>5min) + { + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-retry-on", "5xx"}}; + setup(request_headers); + EXPECT_TRUE(state_->enabled()); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "429"}, + {"X-RateLimit-Reset", "1000000301"}}; + EXPECT_EQ(absl::nullopt, state_->parseResetInterval(response_headers)); + } + + // The only reset header matches (seconds) and the header value is in within range + { + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-retry-on", "5xx"}}; + setup(request_headers); + EXPECT_TRUE(state_->enabled()); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "429"}, {"Retry-After", "300"}}; + EXPECT_EQ(absl::optional(300000), + state_->parseResetInterval(response_headers)); + } + + // The only reset header matches (timestamp) and the header value is in within range + { + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-retry-on", "5xx"}}; + setup(request_headers); + EXPECT_TRUE(state_->enabled()); + + Http::TestResponseHeaderMapImpl response_headers{{":status", "429"}, + {"x-ratelimit-reset", "1000000300"}}; + EXPECT_EQ(absl::optional(300000), + state_->parseResetInterval(response_headers)); + } + + // The second (timestamp) and third (seconds) reset headers match but Retry-After comes first in + // reset_headers so it is used + { + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-retry-on", "5xx"}}; + setup(request_headers); + EXPECT_TRUE(state_->enabled()); + + Http::TestResponseHeaderMapImpl response_headers{ + {":status", "429"}, {"x-ratelimit-reset", "1000000002"}, {"retry-after", "3"}}; + EXPECT_EQ(absl::optional(3000), + state_->parseResetInterval(response_headers)); + } +} + +TEST_F(RouterRetryStateImplTest, RateLimitedRetryBackoffStrategy) { + Protobuf::RepeatedPtrField reset_headers; + auto* reset_header = reset_headers.Add(); + reset_header->set_name("Retry-After"); + reset_header->set_format(envoy::config::route::v3::RetryPolicy::SECONDS); + + policy_.num_retries_ = 3; + policy_.reset_headers_ = ResetHeaderParserImpl::buildResetHeaderParserVector(reset_headers); + + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-retry-on", "5xx"}}; + setup(request_headers); + EXPECT_TRUE(state_->enabled()); + + retry_timer_ = new Event::MockTimer(&dispatcher_); + Http::TestResponseHeaderMapImpl response_headers_reset_1{{":status", "500"}, + {"retry-after", "2"}}; + Http::TestResponseHeaderMapImpl response_headers_plain{{":status", "500"}}; + Http::TestResponseHeaderMapImpl response_headers_reset_2{{":status", "500"}, + {"retry-after", "5"}}; + + // reset header present -> ratelimit backoff used + EXPECT_CALL(random_, random()).WillOnce(Return(190)); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(2190), _)); + EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers_reset_1, callback_)); + EXPECT_CALL(callback_ready_, ready()); + retry_timer_->invokeCallback(); + + // reset header not present -> exponential backoff used + EXPECT_CALL(random_, random()).WillOnce(Return(190)); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(15), _)); + EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers_plain, callback_)); + EXPECT_CALL(callback_ready_, ready()); + retry_timer_->invokeCallback(); + + // reset header present -> ratelimit backoff used + EXPECT_CALL(random_, random()).WillOnce(Return(2190)); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(7190), _)); + EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers_reset_2, callback_)); + EXPECT_CALL(callback_ready_, ready()); + retry_timer_->invokeCallback(); + + EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, + state_->shouldRetryHeaders(response_headers_reset_2, callback_)); + + EXPECT_EQ(2UL, cluster_.stats().upstream_rq_retry_backoff_ratelimited_.value()); + EXPECT_EQ(1UL, cluster_.stats().upstream_rq_retry_backoff_exponential_.value()); +} + TEST_F(RouterRetryStateImplTest, HostSelectionAttempts) { policy_.host_selection_max_attempts_ = 2; policy_.retry_on_ = RetryPolicy::RETRY_ON_CONNECT_FAILURE; diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 0e829d87914e..6cc60aad3c01 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -71,7 +71,7 @@ class RouterTestFilter : public Filter { RetryStatePtr createRetryState(const RetryPolicy&, Http::RequestHeaderMap&, const Upstream::ClusterInfo&, const VirtualCluster*, Runtime::Loader&, Random::RandomGenerator&, Event::Dispatcher&, - Upstream::ResourcePriority) override { + TimeSource&, Upstream::ResourcePriority) override { EXPECT_EQ(nullptr, retry_state_); retry_state_ = new NiceMock(); if (reject_all_hosts_) { diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index 8662e760364a..42ed52b60d83 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -64,7 +64,7 @@ class TestFilter : public Filter { RetryStatePtr createRetryState(const RetryPolicy&, Http::RequestHeaderMap&, const Upstream::ClusterInfo&, const VirtualCluster*, Runtime::Loader&, Random::RandomGenerator&, Event::Dispatcher&, - Upstream::ResourcePriority) override { + TimeSource&, Upstream::ResourcePriority) override { EXPECT_EQ(nullptr, retry_state_); retry_state_ = new NiceMock(); return RetryStatePtr{retry_state_}; diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index c12f94d4e99b..332f9e7651e8 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -757,7 +757,7 @@ TEST(UtilityTest, PrepareDnsRefreshStrategy) { BackOffStrategyPtr strategy = Config::Utility::prepareDnsRefreshStrategy< envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig>(dns_cache_config, 5000, random); - EXPECT_NE(nullptr, dynamic_cast(strategy.get())); + EXPECT_NE(nullptr, dynamic_cast(strategy.get())); } { diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index f66a9b8a14bf..2ffc1ade58c9 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -288,6 +288,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // 2020/07/21 12034 44811 46000 Add configurable histogram buckets. // 2020/07/31 12035 45002 46000 Init manager store unready targets in hash map. // 2020/08/10 12275 44949 46000 Re-organize tls histogram maps to improve continuity. + // 2020/08/11 12202 44949 46500 router: add new retry back-off strategy // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -308,7 +309,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // https://github.com/envoyproxy/envoy/issues/12209 // EXPECT_MEMORY_EQ(m_per_cluster, 44949); } - EXPECT_MEMORY_LE(m_per_cluster, 46000); // Round up to allow platform variations. + EXPECT_MEMORY_LE(m_per_cluster, 46500); // Round up to allow platform variations. } TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { @@ -366,6 +367,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // 2020/07/21 12034 36923 38000 Add configurable histogram buckets. // 2020/07/31 12035 37114 38000 Init manager store unready targets in hash map. // 2020/08/10 12275 37061 38000 Re-organize tls histogram maps to improve continuity. + // 2020/08/11 12202 37061 38500 router: add new retry back-off strategy // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -386,7 +388,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // https://github.com/envoyproxy/envoy/issues/12209 // EXPECT_MEMORY_EQ(m_per_cluster, 37061); } - EXPECT_MEMORY_LE(m_per_cluster, 38000); // Round up to allow platform variations. + EXPECT_MEMORY_LE(m_per_cluster, 38500); // Round up to allow platform variations. } TEST_P(ClusterMemoryTestRunner, MemoryLargeHostSizeWithStats) { diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 890a6ccef893..f0c35d2e72fc 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -119,6 +119,10 @@ class TestRetryPolicy : public RetryPolicy { absl::optional baseInterval() const override { return base_interval_; } absl::optional maxInterval() const override { return max_interval_; } + std::chrono::milliseconds resetMaxInterval() const override { return reset_max_interval_; } + const std::vector& resetHeaders() const override { + return reset_headers_; + } std::chrono::milliseconds per_try_timeout_{0}; uint32_t num_retries_{}; @@ -129,6 +133,8 @@ class TestRetryPolicy : public RetryPolicy { std::vector retriable_request_headers_; absl::optional base_interval_{}; absl::optional max_interval_{}; + std::vector reset_headers_{}; + std::chrono::milliseconds reset_max_interval_{300000}; }; class MockInternalRedirectPolicy : public InternalRedirectPolicy { @@ -157,6 +163,8 @@ class MockRetryState : public RetryState { void expectResetRetry(); MOCK_METHOD(bool, enabled, ()); + MOCK_METHOD(absl::optional, parseResetInterval, + (const Http::ResponseHeaderMap& response_headers), (const)); MOCK_METHOD(RetryStatus, shouldRetryHeaders, (const Http::ResponseHeaderMap& response_headers, DoRetryCallback callback)); MOCK_METHOD(bool, wouldRetryFromHeaders, (const Http::ResponseHeaderMap& response_headers)); From 5f3a35aa4cbe77d329539bb689a5566fc96f8dd5 Mon Sep 17 00:00:00 2001 From: Yifan Yang Date: Thu, 13 Aug 2020 20:15:39 -0400 Subject: [PATCH 59/67] header: getting rid of exception-throwing behaviors in header files [the rest] (#12611) getting rid of exception-throwing behaviors in header files Signed-off-by: Yifan Yang --- source/common/protobuf/utility.cc | 10 +++ source/common/protobuf/utility.h | 64 ++++++++++++++----- source/extensions/common/BUILD | 1 + source/extensions/common/utility.h | 4 +- .../extensions/filters/network/common/BUILD | 1 + .../filters/network/common/factory_base.h | 5 +- .../network/dubbo_proxy/app_exception.h | 4 +- .../filters/network/kafka/serialization.h | 12 ++-- .../filters/network/mongo_proxy/bson_impl.h | 3 +- .../network/thrift_proxy/thrift_object.h | 8 ++- 10 files changed, 84 insertions(+), 28 deletions(-) diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 288c3fc9620a..50de7f88d4e3 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -251,6 +251,16 @@ ProtoValidationException::ProtoValidationException(const std::string& validation ENVOY_LOG_MISC(debug, "Proto validation error; throwing {}", what()); } +void ProtoExceptionUtil::throwMissingFieldException(const std::string& field_name, + const Protobuf::Message& message) { + throw MissingFieldException(field_name, message); +} + +void ProtoExceptionUtil::throwProtoValidationException(const std::string& validation_error, + const Protobuf::Message& message) { + throw ProtoValidationException(validation_error, message); +} + size_t MessageUtil::hash(const Protobuf::Message& message) { std::string text_format; diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index dc2ec54d1863..e6176a416dd4 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -23,10 +23,14 @@ // Obtain the value of a wrapped field (e.g. google.protobuf.UInt32Value) if set. Otherwise, throw // a MissingFieldException. -#define PROTOBUF_GET_WRAPPED_REQUIRED(message, field_name) \ - ((message).has_##field_name() ? (message).field_name().value() \ - : throw MissingFieldException(#field_name, (message))) +#define PROTOBUF_GET_WRAPPED_REQUIRED(message, field_name) \ + ([](const auto& msg) { \ + if (!msg.has_##field_name()) { \ + ::Envoy::ProtoExceptionUtil::throwMissingFieldException(#field_name, msg); \ + } \ + return msg.field_name().value(); \ + }((message))) // Obtain the milliseconds value of a google.protobuf.Duration field if set. Otherwise, return the // default value. #define PROTOBUF_GET_MS_OR_DEFAULT(message, field_name, default_value) \ @@ -48,14 +52,22 @@ // Obtain the milliseconds value of a google.protobuf.Duration field if set. Otherwise, throw a // MissingFieldException. #define PROTOBUF_GET_MS_REQUIRED(message, field_name) \ - ((message).has_##field_name() ? DurationUtil::durationToMilliseconds((message).field_name()) \ - : throw MissingFieldException(#field_name, (message))) + ([](const auto& msg) { \ + if (!msg.has_##field_name()) { \ + ::Envoy::ProtoExceptionUtil::throwMissingFieldException(#field_name, msg); \ + } \ + return DurationUtil::durationToMilliseconds(msg.field_name()); \ + }((message))) // Obtain the seconds value of a google.protobuf.Duration field if set. Otherwise, throw a // MissingFieldException. #define PROTOBUF_GET_SECONDS_REQUIRED(message, field_name) \ - ((message).has_##field_name() ? DurationUtil::durationToSeconds((message).field_name()) \ - : throw MissingFieldException(#field_name, (message))) + ([](const auto& msg) { \ + if (!msg.has_##field_name()) { \ + ::Envoy::ProtoExceptionUtil::throwMissingFieldException(#field_name, msg); \ + } \ + return DurationUtil::durationToSeconds(msg.field_name()); \ + }((message))) namespace Envoy { namespace ProtobufPercentHelper { @@ -90,10 +102,13 @@ uint64_t fractionalPercentDenominatorToInt( // @param field_name supplies the field name in the message. // @param default_value supplies the default if the field is not present. #define PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(message, field_name, default_value) \ - (!std::isnan((message).field_name().value()) \ - ? (message).has_##field_name() ? (message).field_name().value() : default_value \ - : throw EnvoyException(fmt::format("Value not in the range of 0..100 range."))) - + ([](const auto& msg) -> double { \ + if (std::isnan(msg.field_name().value())) { \ + ::Envoy::ExceptionUtil::throwEnvoyException( \ + fmt::format("Value not in the range of 0..100 range.")); \ + } \ + return (msg).has_##field_name() ? (msg).field_name().value() : default_value; \ + }((message))) // Convert an envoy::type::v3::Percent to a rounded integer or a default. // @param message supplies the proto message containing the field. // @param field_name supplies the field name in the message. @@ -104,11 +119,15 @@ uint64_t fractionalPercentDenominatorToInt( // Issue: https://github.com/envoyproxy/protoc-gen-validate/issues/85 #define PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(message, field_name, max_value, \ default_value) \ - (!std::isnan((message).field_name().value()) \ - ? (message).has_##field_name() \ - ? ProtobufPercentHelper::convertPercent((message).field_name().value(), max_value) \ - : ProtobufPercentHelper::checkAndReturnDefault(default_value, max_value) \ - : throw EnvoyException(fmt::format("Value not in the range of 0..100 range."))) + ([](const auto& msg) { \ + if (std::isnan(msg.field_name().value())) { \ + ::Envoy::ExceptionUtil::throwEnvoyException( \ + fmt::format("Value not in the range of 0..100 range.")); \ + } \ + return (msg).has_##field_name() \ + ? ProtobufPercentHelper::convertPercent((msg).field_name().value(), max_value) \ + : ProtobufPercentHelper::checkAndReturnDefault(default_value, max_value); \ + }((message))) namespace Envoy { @@ -185,6 +204,17 @@ class ProtoValidationException : public EnvoyException { ProtoValidationException(const std::string& validation_error, const Protobuf::Message& message); }; +/** + * utility functions to call when throwing exceptions in header files + */ +class ProtoExceptionUtil { +public: + static void throwMissingFieldException(const std::string& field_name, + const Protobuf::Message& message); + static void throwProtoValidationException(const std::string& validation_error, + const Protobuf::Message& message); +}; + class MessageUtil { public: // std::hash @@ -256,7 +286,7 @@ class MessageUtil { std::string err; if (!Validate(message, &err)) { - throw ProtoValidationException(err, API_RECOVER_ORIGINAL(message)); + ProtoExceptionUtil::throwProtoValidationException(err, API_RECOVER_ORIGINAL(message)); } } diff --git a/source/extensions/common/BUILD b/source/extensions/common/BUILD index abc0d81c2d50..50e003b97561 100644 --- a/source/extensions/common/BUILD +++ b/source/extensions/common/BUILD @@ -17,5 +17,6 @@ envoy_cc_library( "//include/envoy/runtime:runtime_interface", "//source/common/common:documentation_url_lib", "//source/common/common:minimal_logger_lib", + "//source/common/common:utility_lib", ], ) diff --git a/source/extensions/common/utility.h b/source/extensions/common/utility.h index 60336fe5e444..378223c912cd 100644 --- a/source/extensions/common/utility.h +++ b/source/extensions/common/utility.h @@ -5,6 +5,7 @@ #include "common/common/documentation_url.h" #include "common/common/logger.h" +#include "common/common/utility.h" namespace Envoy { namespace Extensions { @@ -92,7 +93,8 @@ class ExtensionNameUtil { return; } - throw EnvoyException(fatalMessage(extension_type, deprecated_name, canonical_name)); + ExceptionUtil::throwEnvoyException( + fatalMessage(extension_type, deprecated_name, canonical_name)); } private: diff --git a/source/extensions/filters/network/common/BUILD b/source/extensions/filters/network/common/BUILD index 09249e400050..c8b8dd24f51f 100644 --- a/source/extensions/filters/network/common/BUILD +++ b/source/extensions/filters/network/common/BUILD @@ -15,6 +15,7 @@ envoy_cc_library( visibility = ["//visibility:public"], deps = [ "//include/envoy/server:filter_config_interface", + "//source/extensions/common:utility_lib", ], ) diff --git a/source/extensions/filters/network/common/factory_base.h b/source/extensions/filters/network/common/factory_base.h index cad30901c11e..ba5bcf52159a 100644 --- a/source/extensions/filters/network/common/factory_base.h +++ b/source/extensions/filters/network/common/factory_base.h @@ -4,6 +4,8 @@ #include "envoy/server/transport_socket_config.h" #include "envoy/upstream/upstream.h" +#include "common/common/utility.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -56,7 +58,8 @@ class FactoryBase : public Server::Configuration::NamedNetworkFilterConfigFactor virtual Upstream::ProtocolOptionsConfigConstSharedPtr createProtocolOptionsTyped(const ProtocolOptionsProto&, Server::Configuration::ProtocolOptionsFactoryContext&) { - throw EnvoyException(fmt::format("filter {} does not support protocol options", name_)); + ExceptionUtil::throwEnvoyException( + fmt::format("filter {} does not support protocol options", name_)); } const std::string name_; diff --git a/source/extensions/filters/network/dubbo_proxy/app_exception.h b/source/extensions/filters/network/dubbo_proxy/app_exception.h index f7415018d654..50419b57c680 100644 --- a/source/extensions/filters/network/dubbo_proxy/app_exception.h +++ b/source/extensions/filters/network/dubbo_proxy/app_exception.h @@ -2,6 +2,8 @@ #include "envoy/common/exception.h" +#include "common/common/utility.h" + #include "extensions/filters/network/dubbo_proxy/filters/filter.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" #include "extensions/filters/network/dubbo_proxy/protocol.h" @@ -32,7 +34,7 @@ struct AppExceptionBase : public EnvoyException, metadata.setResponseStatus(status_); metadata.setMessageType(MessageType::Response); if (!protocol.encode(buffer, metadata, what(), response_type_)) { - throw EnvoyException("Failed to encode local reply message"); + ExceptionUtil::throwEnvoyException("Failed to encode local reply message"); } return ResponseType::Exception; diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 8e833e67720d..65a2b18e78e7 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -11,6 +11,7 @@ #include "common/common/byte_order.h" #include "common/common/fmt.h" +#include "common/common/utility.h" #include "extensions/filters/network/kafka/kafka_types.h" @@ -206,7 +207,8 @@ class VarUInt32Deserializer : public Deserializer { offset_ += 7; // Valid input can have at most 5 bytes. if (offset_ >= 5 * 7) { - throw EnvoyException("VarUInt32 is too long (5th byte has highest bit set)"); + ExceptionUtil::throwEnvoyException( + "VarUInt32 is too long (5th byte has highest bit set)"); } } } @@ -463,7 +465,7 @@ class ArrayDeserializer : public Deserializer= 0) { children_ = std::vector(required_); } else { - throw EnvoyException(absl::StrCat("invalid ARRAY length: ", required_)); + ExceptionUtil::throwEnvoyException(absl::StrCat("invalid ARRAY length: ", required_)); } length_consumed_ = true; } @@ -542,7 +544,8 @@ class CompactArrayDeserializer if (required >= 1) { children_ = std::vector(required - 1); } else { - throw EnvoyException(absl::StrCat("invalid COMPACT_ARRAY length: ", required)); + ExceptionUtil::throwEnvoyException( + absl::StrCat("invalid COMPACT_ARRAY length: ", required)); } length_consumed_ = true; } @@ -625,7 +628,8 @@ class NullableArrayDeserializer ready_ = true; } if (required_ < NULL_ARRAY_LENGTH) { - throw EnvoyException(fmt::format("invalid NULLABLE_ARRAY length: {}", required_)); + ExceptionUtil::throwEnvoyException( + fmt::format("invalid NULLABLE_ARRAY length: {}", required_)); } length_consumed_ = true; diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.h b/source/extensions/filters/network/mongo_proxy/bson_impl.h index 71e18b4546fa..57e44c951d44 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.h +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.h @@ -8,6 +8,7 @@ #include "envoy/common/exception.h" #include "common/common/logger.h" +#include "common/common/utility.h" #include "extensions/filters/network/mongo_proxy/bson.h" @@ -153,7 +154,7 @@ class FieldImpl : public Field { private: void checkType(Type type) const { if (type_ != type) { - throw EnvoyException("invalid BSON field type cast"); + ExceptionUtil::throwEnvoyException("invalid BSON field type cast"); } } diff --git a/source/extensions/filters/network/thrift_proxy/thrift_object.h b/source/extensions/filters/network/thrift_proxy/thrift_object.h index 4f99d23bf322..6c4d9f355510 100644 --- a/source/extensions/filters/network/thrift_proxy/thrift_object.h +++ b/source/extensions/filters/network/thrift_proxy/thrift_object.h @@ -6,6 +6,8 @@ #include "envoy/buffer/buffer.h" #include "envoy/common/exception.h" +#include "common/common/utility.h" + #include "extensions/filters/network/thrift_proxy/thrift.h" namespace Envoy { @@ -36,9 +38,9 @@ class ThriftValue { // and throw if the value's type doesn't match. FieldType expected_field_type = Traits::getFieldType(); if (expected_field_type != type()) { - throw EnvoyException(fmt::format("expected field type {}, got {}", - static_cast(expected_field_type), - static_cast(type()))); + ExceptionUtil::throwEnvoyException(fmt::format("expected field type {}, got {}", + static_cast(expected_field_type), + static_cast(type()))); } return *static_cast(getValue()); From b09971dee4643c7ffd8b1a7aca7d9058fd1ef285 Mon Sep 17 00:00:00 2001 From: James Adam Buckland Date: Thu, 13 Aug 2020 20:19:39 -0400 Subject: [PATCH 60/67] [tls] Move handshaking behavior into SslSocketInfo. (#12571) This change makes possible (and simpler) a later change in which we allow users to modify the behavior of the handshaker (i.e. to add new branches for handing SSL_ERRORs) by using an extension point. Signed-off-by: James Buckland --- include/envoy/network/BUILD | 7 + include/envoy/network/post_io_action.h | 17 ++ include/envoy/network/transport_socket.h | 11 +- include/envoy/ssl/BUILD | 17 ++ include/envoy/ssl/handshaker.h | 47 ++++++ include/envoy/ssl/ssl_socket_state.h | 9 ++ .../transport_sockets/common/passthrough.cc | 2 +- .../transport_sockets/common/passthrough.h | 2 +- source/extensions/transport_sockets/tls/BUILD | 2 + .../transport_sockets/tls/ssl_socket.cc | 145 ++++++++++-------- .../transport_sockets/tls/ssl_socket.h | 29 +++- .../proxy_filter_integration_test.cc | 8 +- .../http/router/auto_sni_integration_test.cc | 12 +- .../transport_sockets/tls/ssl_socket_test.cc | 36 ++--- 14 files changed, 231 insertions(+), 113 deletions(-) create mode 100644 include/envoy/network/post_io_action.h create mode 100644 include/envoy/ssl/handshaker.h create mode 100644 include/envoy/ssl/ssl_socket_state.h diff --git a/include/envoy/network/BUILD b/include/envoy/network/BUILD index 3a8e67613c58..6e3bb429fe5b 100644 --- a/include/envoy/network/BUILD +++ b/include/envoy/network/BUILD @@ -130,12 +130,19 @@ envoy_cc_library( hdrs = ["transport_socket.h"], deps = [ ":io_handle_interface", + ":post_io_action_interface", ":proxy_protocol_options_lib", "//include/envoy/buffer:buffer_interface", "//include/envoy/ssl:connection_interface", ], ) +envoy_cc_library( + name = "post_io_action_interface", + hdrs = ["post_io_action.h"], + deps = [], +) + envoy_cc_library( name = "connection_balancer_interface", hdrs = ["connection_balancer.h"], diff --git a/include/envoy/network/post_io_action.h b/include/envoy/network/post_io_action.h new file mode 100644 index 000000000000..3b828bc1d5e7 --- /dev/null +++ b/include/envoy/network/post_io_action.h @@ -0,0 +1,17 @@ +#pragma once + +namespace Envoy { +namespace Network { + +/** + * Action that should occur on a connection after I/O. + */ +enum class PostIoAction { + // Close the connection. + Close, + // Keep the connection open. + KeepOpen +}; + +} // namespace Network +} // namespace Envoy diff --git a/include/envoy/network/transport_socket.h b/include/envoy/network/transport_socket.h index 9e117b116134..fe054ce2f16d 100644 --- a/include/envoy/network/transport_socket.h +++ b/include/envoy/network/transport_socket.h @@ -5,6 +5,7 @@ #include "envoy/buffer/buffer.h" #include "envoy/common/pure.h" #include "envoy/network/io_handle.h" +#include "envoy/network/post_io_action.h" #include "envoy/network/proxy_protocol.h" #include "envoy/ssl/connection.h" @@ -16,16 +17,6 @@ namespace Network { class Connection; enum class ConnectionEvent; -/** - * Action that should occur on a connection after I/O. - */ -enum class PostIoAction { - // Close the connection. - Close, - // Keep the connection open. - KeepOpen -}; - /** * Result of each I/O event. */ diff --git a/include/envoy/ssl/BUILD b/include/envoy/ssl/BUILD index b8e7d530174f..b295a20e2a1a 100644 --- a/include/envoy/ssl/BUILD +++ b/include/envoy/ssl/BUILD @@ -13,6 +13,7 @@ envoy_cc_library( hdrs = ["connection.h"], external_deps = ["abseil_optional"], deps = [ + ":ssl_socket_state", "//include/envoy/common:time_interface", ], ) @@ -68,3 +69,19 @@ envoy_cc_library( deps = [ ], ) + +envoy_cc_library( + name = "ssl_socket_state", + hdrs = ["ssl_socket_state.h"], + deps = [], +) + +envoy_cc_library( + name = "handshaker_interface", + hdrs = ["handshaker.h"], + external_deps = ["ssl"], + deps = [ + "//include/envoy/network:connection_interface", + "//include/envoy/network:post_io_action_interface", + ], +) diff --git a/include/envoy/ssl/handshaker.h b/include/envoy/ssl/handshaker.h new file mode 100644 index 000000000000..de11fc85f41f --- /dev/null +++ b/include/envoy/ssl/handshaker.h @@ -0,0 +1,47 @@ +#pragma once + +#include "envoy/network/connection.h" +#include "envoy/network/post_io_action.h" + +#include "openssl/ssl.h" + +namespace Envoy { +namespace Ssl { + +class HandshakeCallbacks { +public: + virtual ~HandshakeCallbacks() = default; + + /** + * @return the connection state. + */ + virtual Network::Connection::State connectionState() const PURE; + + /** + * A callback which will be executed at most once upon successful completion + * of a handshake. + */ + virtual void onSuccess(SSL* ssl) PURE; + + /** + * A callback which will be executed at most once upon handshake failure. + */ + virtual void onFailure() PURE; +}; + +/** + * Base interface for performing TLS handshakes. + */ +class Handshaker { +public: + virtual ~Handshaker() = default; + + /** + * Performs a TLS handshake and returns an action indicating + * whether the callsite should close the connection or keep it open. + */ + virtual Network::PostIoAction doHandshake() PURE; +}; + +} // namespace Ssl +} // namespace Envoy diff --git a/include/envoy/ssl/ssl_socket_state.h b/include/envoy/ssl/ssl_socket_state.h new file mode 100644 index 000000000000..aa60fbc178ab --- /dev/null +++ b/include/envoy/ssl/ssl_socket_state.h @@ -0,0 +1,9 @@ +#pragma once + +namespace Envoy { +namespace Ssl { + +enum class SocketState { PreHandshake, HandshakeInProgress, HandshakeComplete, ShutdownSent }; + +} // namespace Ssl +} // namespace Envoy diff --git a/source/extensions/transport_sockets/common/passthrough.cc b/source/extensions/transport_sockets/common/passthrough.cc index 60d632adb24a..86fc282ed3e3 100644 --- a/source/extensions/transport_sockets/common/passthrough.cc +++ b/source/extensions/transport_sockets/common/passthrough.cc @@ -44,4 +44,4 @@ Ssl::ConnectionInfoConstSharedPtr PassthroughSocket::ssl() const { } // namespace TransportSockets } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/extensions/transport_sockets/common/passthrough.h b/source/extensions/transport_sockets/common/passthrough.h index bbf832c73419..5084d973a865 100644 --- a/source/extensions/transport_sockets/common/passthrough.h +++ b/source/extensions/transport_sockets/common/passthrough.h @@ -29,4 +29,4 @@ class PassthroughSocket : public Network::TransportSocket { } // namespace TransportSockets } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/BUILD b/source/extensions/transport_sockets/tls/BUILD index 1cd091050d15..e12b887d01ff 100644 --- a/source/extensions/transport_sockets/tls/BUILD +++ b/source/extensions/transport_sockets/tls/BUILD @@ -47,7 +47,9 @@ envoy_cc_library( ":utility_lib", "//include/envoy/network:connection_interface", "//include/envoy/network:transport_socket_interface", + "//include/envoy/ssl:handshaker_interface", "//include/envoy/ssl:ssl_socket_extended_info_interface", + "//include/envoy/ssl:ssl_socket_state", "//include/envoy/ssl/private_key:private_key_callbacks_interface", "//include/envoy/ssl/private_key:private_key_interface", "//include/envoy/stats:stats_macros", diff --git a/source/extensions/transport_sockets/tls/ssl_socket.cc b/source/extensions/transport_sockets/tls/ssl_socket.cc index ab2644ccc808..a5f1329191e8 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.cc +++ b/source/extensions/transport_sockets/tls/ssl_socket.cc @@ -45,9 +45,9 @@ class NotReadySslSocket : public Network::TransportSocket { SslSocket::SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, const Network::TransportSocketOptionsSharedPtr& transport_socket_options) : transport_socket_options_(transport_socket_options), - ctx_(std::dynamic_pointer_cast(ctx)), state_(SocketState::PreHandshake) { + ctx_(std::dynamic_pointer_cast(ctx)) { bssl::UniquePtr ssl = ctx_->newSsl(transport_socket_options_.get()); - info_ = std::make_shared(std::move(ssl), ctx_); + info_ = std::make_shared(std::move(ssl), ctx_, this); if (state == InitialState::Client) { SSL_set_connect_state(rawSsl()); @@ -96,9 +96,10 @@ SslSocket::ReadResult SslSocket::sslReadIntoSlice(Buffer::RawSlice& slice) { } Network::IoResult SslSocket::doRead(Buffer::Instance& read_buffer) { - if (state_ != SocketState::HandshakeComplete && state_ != SocketState::ShutdownSent) { + if (info_->state() != Ssl::SocketState::HandshakeComplete && + info_->state() != Ssl::SocketState::ShutdownSent) { PostIoAction action = doHandshake(); - if (action == PostIoAction::Close || state_ != SocketState::HandshakeComplete) { + if (action == PostIoAction::Close || info_->state() != Ssl::SocketState::HandshakeComplete) { // end_stream is false because either a hard error occurred (action == Close) or // the handshake isn't complete, so a half-close cannot occur yet. return {action, 0, false}; @@ -158,7 +159,7 @@ Network::IoResult SslSocket::doRead(Buffer::Instance& read_buffer) { void SslSocket::onPrivateKeyMethodComplete() { ASSERT(isThreadSafe()); - ASSERT(state_ == SocketState::HandshakeInProgress); + ASSERT(info_->state() == Ssl::SocketState::HandshakeInProgress); // Resume handshake. PostIoAction action = doHandshake(); @@ -168,39 +169,19 @@ void SslSocket::onPrivateKeyMethodComplete() { } } -PostIoAction SslSocket::doHandshake() { - ASSERT(state_ != SocketState::HandshakeComplete && state_ != SocketState::ShutdownSent); - int rc = SSL_do_handshake(rawSsl()); - if (rc == 1) { - ENVOY_CONN_LOG(debug, "handshake complete", callbacks_->connection()); - state_ = SocketState::HandshakeComplete; - ctx_->logHandshake(rawSsl()); - callbacks_->raiseEvent(Network::ConnectionEvent::Connected); +Network::Connection::State SslSocket::connectionState() const { + return callbacks_->connection().state(); +} - // It's possible that we closed during the handshake callback. - return callbacks_->connection().state() == Network::Connection::State::Open - ? PostIoAction::KeepOpen - : PostIoAction::Close; - } else { - int err = SSL_get_error(rawSsl(), rc); - switch (err) { - case SSL_ERROR_WANT_READ: - case SSL_ERROR_WANT_WRITE: - ENVOY_CONN_LOG(debug, "handshake expecting {}", callbacks_->connection(), - err == SSL_ERROR_WANT_READ ? "read" : "write"); - return PostIoAction::KeepOpen; - case SSL_ERROR_WANT_PRIVATE_KEY_OPERATION: - ENVOY_CONN_LOG(debug, "handshake continued asynchronously", callbacks_->connection()); - state_ = SocketState::HandshakeInProgress; - return PostIoAction::KeepOpen; - default: - ENVOY_CONN_LOG(debug, "handshake error: {}", callbacks_->connection(), err); - drainErrorQueue(); - return PostIoAction::Close; - } - } +void SslSocket::onSuccess(SSL* ssl) { + ctx_->logHandshake(ssl); + callbacks_->raiseEvent(Network::ConnectionEvent::Connected); } +void SslSocket::onFailure() { drainErrorQueue(); } + +PostIoAction SslSocket::doHandshake() { return info_->doHandshake(); } + void SslSocket::drainErrorQueue() { bool saw_error = false; bool saw_counted_error = false; @@ -229,10 +210,11 @@ void SslSocket::drainErrorQueue() { } Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_stream) { - ASSERT(state_ != SocketState::ShutdownSent || write_buffer.length() == 0); - if (state_ != SocketState::HandshakeComplete && state_ != SocketState::ShutdownSent) { + ASSERT(info_->state() != Ssl::SocketState::ShutdownSent || write_buffer.length() == 0); + if (info_->state() != Ssl::SocketState::HandshakeComplete && + info_->state() != Ssl::SocketState::ShutdownSent) { PostIoAction action = doHandshake(); - if (action == PostIoAction::Close || state_ != SocketState::HandshakeComplete) { + if (action == PostIoAction::Close || info_->state() != Ssl::SocketState::HandshakeComplete) { return {action, 0, false}; } } @@ -285,18 +267,18 @@ Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_st return {PostIoAction::KeepOpen, total_bytes_written, false}; } -void SslSocket::onConnected() { ASSERT(state_ == SocketState::PreHandshake); } +void SslSocket::onConnected() { ASSERT(info_->state() == Ssl::SocketState::PreHandshake); } Ssl::ConnectionInfoConstSharedPtr SslSocket::ssl() const { return info_; } void SslSocket::shutdownSsl() { - ASSERT(state_ != SocketState::PreHandshake); - if (state_ != SocketState::ShutdownSent && + ASSERT(info_->state() != Ssl::SocketState::PreHandshake); + if (info_->state() != Ssl::SocketState::ShutdownSent && callbacks_->connection().state() != Network::Connection::State::Closed) { int rc = SSL_shutdown(rawSsl()); ENVOY_CONN_LOG(debug, "SSL shutdown: rc={}", callbacks_->connection(), rc); drainErrorQueue(); - state_ = SocketState::ShutdownSent; + info_->setState(Ssl::SocketState::ShutdownSent); } } @@ -309,22 +291,24 @@ Envoy::Ssl::ClientValidationStatus SslExtendedSocketInfoImpl::certificateValidat return certificate_validation_status_; } -SslSocketInfo::SslSocketInfo(bssl::UniquePtr ssl, ContextImplSharedPtr ctx) - : ssl_(std::move(ssl)) { +SslHandshakerImpl::SslHandshakerImpl(bssl::UniquePtr ssl, ContextImplSharedPtr ctx, + Ssl::HandshakeCallbacks* handshake_callbacks) + : ssl_(std::move(ssl)), handshake_callbacks_(handshake_callbacks), + state_(Ssl::SocketState::PreHandshake) { SSL_set_ex_data(ssl_.get(), ctx->sslExtendedSocketInfoIndex(), &(this->extended_socket_info_)); } -bool SslSocketInfo::peerCertificatePresented() const { +bool SslHandshakerImpl::peerCertificatePresented() const { bssl::UniquePtr cert(SSL_get_peer_certificate(ssl())); return cert != nullptr; } -bool SslSocketInfo::peerCertificateValidated() const { +bool SslHandshakerImpl::peerCertificateValidated() const { return extended_socket_info_.certificateValidationStatus() == Envoy::Ssl::ClientValidationStatus::Validated; } -absl::Span SslSocketInfo::uriSanLocalCertificate() const { +absl::Span SslHandshakerImpl::uriSanLocalCertificate() const { if (!cached_uri_san_local_certificate_.empty()) { return cached_uri_san_local_certificate_; } @@ -339,7 +323,7 @@ absl::Span SslSocketInfo::uriSanLocalCertificate() const { return cached_uri_san_local_certificate_; } -absl::Span SslSocketInfo::dnsSansLocalCertificate() const { +absl::Span SslHandshakerImpl::dnsSansLocalCertificate() const { if (!cached_dns_san_local_certificate_.empty()) { return cached_dns_san_local_certificate_; } @@ -353,7 +337,7 @@ absl::Span SslSocketInfo::dnsSansLocalCertificate() const { return cached_dns_san_local_certificate_; } -const std::string& SslSocketInfo::sha256PeerCertificateDigest() const { +const std::string& SslHandshakerImpl::sha256PeerCertificateDigest() const { if (!cached_sha_256_peer_certificate_digest_.empty()) { return cached_sha_256_peer_certificate_digest_; } @@ -371,7 +355,7 @@ const std::string& SslSocketInfo::sha256PeerCertificateDigest() const { return cached_sha_256_peer_certificate_digest_; } -const std::string& SslSocketInfo::sha1PeerCertificateDigest() const { +const std::string& SslHandshakerImpl::sha1PeerCertificateDigest() const { if (!cached_sha_1_peer_certificate_digest_.empty()) { return cached_sha_1_peer_certificate_digest_; } @@ -389,7 +373,7 @@ const std::string& SslSocketInfo::sha1PeerCertificateDigest() const { return cached_sha_1_peer_certificate_digest_; } -const std::string& SslSocketInfo::urlEncodedPemEncodedPeerCertificate() const { +const std::string& SslHandshakerImpl::urlEncodedPemEncodedPeerCertificate() const { if (!cached_url_encoded_pem_encoded_peer_certificate_.empty()) { return cached_url_encoded_pem_encoded_peer_certificate_; } @@ -411,7 +395,7 @@ const std::string& SslSocketInfo::urlEncodedPemEncodedPeerCertificate() const { return cached_url_encoded_pem_encoded_peer_certificate_; } -const std::string& SslSocketInfo::urlEncodedPemEncodedPeerCertificateChain() const { +const std::string& SslHandshakerImpl::urlEncodedPemEncodedPeerCertificateChain() const { if (!cached_url_encoded_pem_encoded_peer_cert_chain_.empty()) { return cached_url_encoded_pem_encoded_peer_cert_chain_; } @@ -441,7 +425,7 @@ const std::string& SslSocketInfo::urlEncodedPemEncodedPeerCertificateChain() con return cached_url_encoded_pem_encoded_peer_cert_chain_; } -absl::Span SslSocketInfo::uriSanPeerCertificate() const { +absl::Span SslHandshakerImpl::uriSanPeerCertificate() const { if (!cached_uri_san_peer_certificate_.empty()) { return cached_uri_san_peer_certificate_; } @@ -455,7 +439,7 @@ absl::Span SslSocketInfo::uriSanPeerCertificate() const { return cached_uri_san_peer_certificate_; } -absl::Span SslSocketInfo::dnsSansPeerCertificate() const { +absl::Span SslHandshakerImpl::dnsSansPeerCertificate() const { if (!cached_dns_san_peer_certificate_.empty()) { return cached_dns_san_peer_certificate_; } @@ -478,7 +462,8 @@ void SslSocket::closeSocket(Network::ConnectionEvent) { // Attempt to send a shutdown before closing the socket. It's possible this won't go out if // there is no room on the socket. We can extend the state machine to handle this at some point // if needed. - if (state_ == SocketState::HandshakeInProgress || state_ == SocketState::HandshakeComplete) { + if (info_->state() == Ssl::SocketState::HandshakeInProgress || + info_->state() == Ssl::SocketState::HandshakeComplete) { shutdownSsl(); } } @@ -490,7 +475,7 @@ std::string SslSocket::protocol() const { return std::string(reinterpret_cast(proto), proto_len); } -uint16_t SslSocketInfo::ciphersuiteId() const { +uint16_t SslHandshakerImpl::ciphersuiteId() const { const SSL_CIPHER* cipher = SSL_get_current_cipher(ssl()); if (cipher == nullptr) { return 0xffff; @@ -502,7 +487,7 @@ uint16_t SslSocketInfo::ciphersuiteId() const { return static_cast(SSL_CIPHER_get_id(cipher)); } -std::string SslSocketInfo::ciphersuiteString() const { +std::string SslHandshakerImpl::ciphersuiteString() const { const SSL_CIPHER* cipher = SSL_get_current_cipher(ssl()); if (cipher == nullptr) { return {}; @@ -511,7 +496,7 @@ std::string SslSocketInfo::ciphersuiteString() const { return SSL_CIPHER_get_name(cipher); } -const std::string& SslSocketInfo::tlsVersion() const { +const std::string& SslHandshakerImpl::tlsVersion() const { if (!cached_tls_version_.empty()) { return cached_tls_version_; } @@ -519,7 +504,8 @@ const std::string& SslSocketInfo::tlsVersion() const { return cached_tls_version_; } -absl::optional SslSocketInfo::x509Extension(absl::string_view extension_name) const { +absl::optional +SslHandshakerImpl::x509Extension(absl::string_view extension_name) const { bssl::UniquePtr cert(SSL_get_peer_certificate(ssl())); if (!cert) { return absl::nullopt; @@ -527,9 +513,36 @@ absl::optional SslSocketInfo::x509Extension(absl::string_view exten return Utility::getX509ExtensionValue(*cert, extension_name); } +Network::PostIoAction SslHandshakerImpl::doHandshake() { + ASSERT(state_ != Ssl::SocketState::HandshakeComplete && state_ != Ssl::SocketState::ShutdownSent); + int rc = SSL_do_handshake(ssl()); + if (rc == 1) { + state_ = Ssl::SocketState::HandshakeComplete; + handshake_callbacks_->onSuccess(ssl()); + + // It's possible that we closed during the handshake callback. + return handshake_callbacks_->connectionState() == Network::Connection::State::Open + ? PostIoAction::KeepOpen + : PostIoAction::Close; + } else { + int err = SSL_get_error(ssl(), rc); + switch (err) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + return PostIoAction::KeepOpen; + case SSL_ERROR_WANT_PRIVATE_KEY_OPERATION: + state_ = Ssl::SocketState::HandshakeInProgress; + return PostIoAction::KeepOpen; + default: + handshake_callbacks_->onFailure(); + return PostIoAction::Close; + } + } +} + absl::string_view SslSocket::failureReason() const { return failure_reason_; } -const std::string& SslSocketInfo::serialNumberPeerCertificate() const { +const std::string& SslHandshakerImpl::serialNumberPeerCertificate() const { if (!cached_serial_number_peer_certificate_.empty()) { return cached_serial_number_peer_certificate_; } @@ -542,7 +555,7 @@ const std::string& SslSocketInfo::serialNumberPeerCertificate() const { return cached_serial_number_peer_certificate_; } -const std::string& SslSocketInfo::issuerPeerCertificate() const { +const std::string& SslHandshakerImpl::issuerPeerCertificate() const { if (!cached_issuer_peer_certificate_.empty()) { return cached_issuer_peer_certificate_; } @@ -555,7 +568,7 @@ const std::string& SslSocketInfo::issuerPeerCertificate() const { return cached_issuer_peer_certificate_; } -const std::string& SslSocketInfo::subjectPeerCertificate() const { +const std::string& SslHandshakerImpl::subjectPeerCertificate() const { if (!cached_subject_peer_certificate_.empty()) { return cached_subject_peer_certificate_; } @@ -568,7 +581,7 @@ const std::string& SslSocketInfo::subjectPeerCertificate() const { return cached_subject_peer_certificate_; } -const std::string& SslSocketInfo::subjectLocalCertificate() const { +const std::string& SslHandshakerImpl::subjectLocalCertificate() const { if (!cached_subject_local_certificate_.empty()) { return cached_subject_local_certificate_; } @@ -581,7 +594,7 @@ const std::string& SslSocketInfo::subjectLocalCertificate() const { return cached_subject_local_certificate_; } -absl::optional SslSocketInfo::validFromPeerCertificate() const { +absl::optional SslHandshakerImpl::validFromPeerCertificate() const { bssl::UniquePtr cert(SSL_get_peer_certificate(ssl())); if (!cert) { return absl::nullopt; @@ -589,7 +602,7 @@ absl::optional SslSocketInfo::validFromPeerCertificate() const { return Utility::getValidFrom(*cert); } -absl::optional SslSocketInfo::expirationPeerCertificate() const { +absl::optional SslHandshakerImpl::expirationPeerCertificate() const { bssl::UniquePtr cert(SSL_get_peer_certificate(ssl())); if (!cert) { return absl::nullopt; @@ -597,7 +610,7 @@ absl::optional SslSocketInfo::expirationPeerCertificate() const { return Utility::getExpirationTime(*cert); } -const std::string& SslSocketInfo::sessionId() const { +const std::string& SslHandshakerImpl::sessionId() const { if (!cached_session_id_.empty()) { return cached_session_id_; } diff --git a/source/extensions/transport_sockets/tls/ssl_socket.h b/source/extensions/transport_sockets/tls/ssl_socket.h index 27416ce7f635..425e6644b209 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.h +++ b/source/extensions/transport_sockets/tls/ssl_socket.h @@ -6,8 +6,10 @@ #include "envoy/network/connection.h" #include "envoy/network/transport_socket.h" #include "envoy/secret/secret_callbacks.h" +#include "envoy/ssl/handshaker.h" #include "envoy/ssl/private_key/private_key_callbacks.h" #include "envoy/ssl/ssl_socket_extended_info.h" +#include "envoy/ssl/ssl_socket_state.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" @@ -39,7 +41,6 @@ struct SslSocketFactoryStats { }; enum class InitialState { Client, Server }; -enum class SocketState { PreHandshake, HandshakeInProgress, HandshakeComplete, ShutdownSent }; class SslExtendedSocketInfoImpl : public Envoy::Ssl::SslExtendedSocketInfo { public: @@ -51,9 +52,10 @@ class SslExtendedSocketInfoImpl : public Envoy::Ssl::SslExtendedSocketInfo { Envoy::Ssl::ClientValidationStatus::NotValidated}; }; -class SslSocketInfo : public Envoy::Ssl::ConnectionInfo { +class SslHandshakerImpl : public Envoy::Ssl::ConnectionInfo, public Envoy::Ssl::Handshaker { public: - SslSocketInfo(bssl::UniquePtr ssl, ContextImplSharedPtr ctx); + SslHandshakerImpl(bssl::UniquePtr ssl, ContextImplSharedPtr ctx, + Ssl::HandshakeCallbacks* handshake_callbacks); // Ssl::ConnectionInfo bool peerCertificatePresented() const override; @@ -77,11 +79,20 @@ class SslSocketInfo : public Envoy::Ssl::ConnectionInfo { std::string ciphersuiteString() const override; const std::string& tlsVersion() const override; absl::optional x509Extension(absl::string_view extension_name) const override; + + // Ssl::Handshaker + Network::PostIoAction doHandshake() override; + + Ssl::SocketState state() { return state_; } + void setState(Ssl::SocketState state) { state_ = state; } SSL* ssl() const { return ssl_.get(); } bssl::UniquePtr ssl_; private: + Ssl::HandshakeCallbacks* handshake_callbacks_; + + Ssl::SocketState state_; mutable std::vector cached_uri_san_local_certificate_; mutable std::string cached_sha_256_peer_certificate_digest_; mutable std::string cached_sha_1_peer_certificate_digest_; @@ -99,10 +110,11 @@ class SslSocketInfo : public Envoy::Ssl::ConnectionInfo { mutable SslExtendedSocketInfoImpl extended_socket_info_; }; -using SslSocketInfoConstSharedPtr = std::shared_ptr; +using SslHandshakerImplSharedPtr = std::shared_ptr; class SslSocket : public Network::TransportSocket, public Envoy::Ssl::PrivateKeyConnectionCallbacks, + public Ssl::HandshakeCallbacks, protected Logger::Loggable { public: SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, @@ -112,7 +124,7 @@ class SslSocket : public Network::TransportSocket, void setTransportSocketCallbacks(Network::TransportSocketCallbacks& callbacks) override; std::string protocol() const override; absl::string_view failureReason() const override; - bool canFlushClose() override { return state_ == SocketState::HandshakeComplete; } + bool canFlushClose() override { return info_->state() == Ssl::SocketState::HandshakeComplete; } void closeSocket(Network::ConnectionEvent close_type) override; Network::IoResult doRead(Buffer::Instance& read_buffer) override; Network::IoResult doWrite(Buffer::Instance& write_buffer, bool end_stream) override; @@ -120,6 +132,10 @@ class SslSocket : public Network::TransportSocket, Ssl::ConnectionInfoConstSharedPtr ssl() const override; // Ssl::PrivateKeyConnectionCallbacks void onPrivateKeyMethodComplete() override; + // Ssl::HandshakeCallbacks + Network::Connection::State connectionState() const override; + void onSuccess(SSL* ssl) override; + void onFailure() override; SSL* rawSslForTest() const { return rawSsl(); } @@ -145,9 +161,8 @@ class SslSocket : public Network::TransportSocket, ContextImplSharedPtr ctx_; uint64_t bytes_to_retry_{}; std::string failure_reason_; - SocketState state_; - SslSocketInfoConstSharedPtr info_; + SslHandshakerImplSharedPtr info_; }; class ClientSslSocketFactory : public Network::TransportSocketFactory, diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index e066cf482805..58efa9bdac2d 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -270,8 +270,8 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTls) { auto response = codec_client_->makeHeaderOnlyRequest(request_headers); waitForNextUpstreamRequest(); - const Extensions::TransportSockets::Tls::SslSocketInfo* ssl_socket = - dynamic_cast( + const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = + dynamic_cast( fake_upstream_connection_->connection().ssl().get()); EXPECT_STREQ("localhost", SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); @@ -295,8 +295,8 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTlsWithIpHost) { waitForNextUpstreamRequest(); // No SNI for IP hosts. - const Extensions::TransportSockets::Tls::SslSocketInfo* ssl_socket = - dynamic_cast( + const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = + dynamic_cast( fake_upstream_connection_->connection().ssl().get()); EXPECT_STREQ(nullptr, SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); diff --git a/test/extensions/filters/http/router/auto_sni_integration_test.cc b/test/extensions/filters/http/router/auto_sni_integration_test.cc index 10f0d7818e3f..d180c8cad956 100644 --- a/test/extensions/filters/http/router/auto_sni_integration_test.cc +++ b/test/extensions/filters/http/router/auto_sni_integration_test.cc @@ -76,8 +76,8 @@ TEST_P(AutoSniIntegrationTest, BasicAutoSniTest) { EXPECT_TRUE(upstream_request_->complete()); EXPECT_TRUE(response_->complete()); - const Extensions::TransportSockets::Tls::SslSocketInfo* ssl_socket = - dynamic_cast( + const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = + dynamic_cast( fake_upstream_connection_->connection().ssl().get()); EXPECT_STREQ("localhost", SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); } @@ -93,8 +93,8 @@ TEST_P(AutoSniIntegrationTest, PassingNotDNS) { EXPECT_TRUE(upstream_request_->complete()); EXPECT_TRUE(response_->complete()); - const Extensions::TransportSockets::Tls::SslSocketInfo* ssl_socket = - dynamic_cast( + const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = + dynamic_cast( fake_upstream_connection_->connection().ssl().get()); EXPECT_STREQ(nullptr, SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); } @@ -112,8 +112,8 @@ TEST_P(AutoSniIntegrationTest, PassingHostWithoutPort) { EXPECT_TRUE(upstream_request_->complete()); EXPECT_TRUE(response_->complete()); - const Extensions::TransportSockets::Tls::SslSocketInfo* ssl_socket = - dynamic_cast( + const Extensions::TransportSockets::Tls::SslHandshakerImpl* ssl_socket = + dynamic_cast( fake_upstream_connection_->connection().ssl().get()); EXPECT_STREQ("example.com", SSL_get_servername(ssl_socket->ssl(), TLSEXT_NAMETYPE_host_name)); } diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 6fa102a74ad4..6bc9c7b9bd8c 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -629,8 +629,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { client_ssl_socket_factory.createTransportSocket(options.transportSocketOptions()), nullptr); if (!options.clientSession().empty()) { - const SslSocketInfo* ssl_socket = - dynamic_cast(client_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL* client_ssl_socket = ssl_socket->ssl(); SSL_CTX* client_ssl_context = SSL_get_SSL_CTX(client_ssl_socket); SSL_SESSION* client_ssl_session = @@ -672,8 +672,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { EXPECT_EQ(options.expectedALPNProtocol(), client_connection->nextProtocol()); } EXPECT_EQ(options.expectedClientCertUri(), server_connection->ssl()->uriSanPeerCertificate()); - const SslSocketInfo* ssl_socket = - dynamic_cast(client_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL* client_ssl_socket = ssl_socket->ssl(); if (!options.expectedProtocolVersion().empty()) { // Assert twice to ensure a cached value is returned and still valid. @@ -689,8 +689,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { } absl::optional server_ssl_requested_server_name; - const SslSocketInfo* server_ssl_socket = - dynamic_cast(server_connection->ssl().get()); + const SslHandshakerImpl* server_ssl_socket = + dynamic_cast(server_connection->ssl().get()); SSL* server_ssl = server_ssl_socket->ssl(); auto requested_server_name = SSL_get_servername(server_ssl, TLSEXT_NAMETYPE_host_name); if (requested_server_name != nullptr) { @@ -2565,8 +2565,8 @@ TEST_P(SslSocketTest, ClientAuthMultipleCAs) { ssl_socket_factory.createTransportSocket(nullptr), nullptr); // Verify that server sent list with 2 acceptable client certificate CA names. - const SslSocketInfo* ssl_socket = - dynamic_cast(client_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL_set_cert_cb( ssl_socket->ssl(), [](SSL* ssl, void*) -> int { @@ -2681,8 +2681,8 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::Connected)) .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { - const SslSocketInfo* ssl_socket = - dynamic_cast(client_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(client_connection->ssl().get()); ssl_session = SSL_get1_session(ssl_socket->ssl()); EXPECT_TRUE(SSL_SESSION_is_resumable(ssl_session)); if (expected_lifetime_hint) { @@ -2704,8 +2704,8 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, socket2->localAddress(), Network::Address::InstanceConstSharedPtr(), ssl_socket_factory.createTransportSocket(nullptr), nullptr); client_connection->addConnectionCallbacks(client_connection_callbacks); - const SslSocketInfo* ssl_socket = - dynamic_cast(client_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL_set_session(ssl_socket->ssl(), ssl_session); SSL_SESSION_free(ssl_session); @@ -2811,8 +2811,8 @@ void testSupportForStatelessSessionResumption(const std::string& server_ctx_yaml std::move(socket), server_ssl_socket_factory.createTransportSocket(nullptr), stream_info); - const SslSocketInfo* ssl_socket = - dynamic_cast(server_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(server_connection->ssl().get()); SSL* server_ssl_socket = ssl_socket->ssl(); SSL_CTX* server_ssl_context = SSL_get_SSL_CTX(server_ssl_socket); if (expect_support) { @@ -3267,8 +3267,8 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) { EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::LocalClose)); EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::Connected)) .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { - const SslSocketInfo* ssl_socket = - dynamic_cast(client_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(client_connection->ssl().get()); ssl_session = SSL_get1_session(ssl_socket->ssl()); EXPECT_TRUE(SSL_SESSION_is_resumable(ssl_session)); server_connection->close(Network::ConnectionCloseType::NoFlush); @@ -3286,8 +3286,8 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) { socket2->localAddress(), Network::Address::InstanceConstSharedPtr(), ssl_socket_factory.createTransportSocket(nullptr), nullptr); client_connection->addConnectionCallbacks(client_connection_callbacks); - const SslSocketInfo* ssl_socket = - dynamic_cast(client_connection->ssl().get()); + const SslHandshakerImpl* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL_set_session(ssl_socket->ssl(), ssl_session); SSL_SESSION_free(ssl_session); From a42a67773c74dc288c960563f21b6d89eb19fa6e Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Thu, 13 Aug 2020 19:45:34 -0700 Subject: [PATCH 61/67] testing: fix multiple race conditions in simulated time tests (#12527) This PR fixes multiple race conditions in tests. The summary is: 1) All waitFor() operations are now fully synchronized. 2) waitFor() no longer moves simulated time and performs real sleeps in all time systems. This means that all network operations are now "instantaneous" and makes all time advances for alarms explicit. This required fixes in a few tests but should make simulated time much easier to reason about. 3) All timeout durations for network operations use real time for timeouts. Signed-off-by: Matt Klein --- .../substitution_formatter_fuzz_test.cc | 4 +- test/common/router/header_parser_fuzz_test.cc | 4 +- .../common/expr/evaluator_fuzz_test.cc | 2 +- .../fault/fault_filter_integration_test.cc | 12 +- .../ratelimit/ratelimit_integration_test.cc | 2 +- .../local_ratelimit_integration_test.cc | 2 +- test/fuzz/utility.h | 26 +- test/integration/BUILD | 5 +- test/integration/autonomous_upstream.cc | 2 +- test/integration/autonomous_upstream.h | 4 +- .../cluster_filter_integration_test.cc | 2 +- test/integration/cx_limit_integration_test.cc | 3 +- test/integration/fake_upstream.cc | 302 +++++++----------- test/integration/fake_upstream.h | 207 ++++++------ .../filter_manager_integration_test.cc | 2 +- test/integration/h1_fuzz.cc | 2 +- test/integration/h2_fuzz.cc | 2 +- test/integration/hds_integration_test.cc | 4 +- test/integration/http2_integration_test.cc | 3 +- test/integration/http_integration.cc | 5 +- .../http_timeout_integration_test.cc | 14 +- test/integration/integration.cc | 32 +- test/integration/integration.h | 4 +- test/integration/integration_test.cc | 2 +- .../load_stats_integration_test.cc | 4 +- test/integration/redirect_integration_test.cc | 4 +- test/integration/server.h | 9 +- test/integration/server_stats.h | 5 +- .../integration/tcp_proxy_integration_test.cc | 26 +- .../tcp_tunneling_integration_test.cc | 4 +- test/integration/utility.cc | 4 +- test/integration/vhds_integration_test.cc | 4 +- test/mocks/common.h | 11 +- test/test_common/BUILD | 3 +- test/test_common/simulated_time_system.cc | 53 +-- test/test_common/simulated_time_system.h | 9 +- .../test_common/simulated_time_system_test.cc | 90 +++--- test/test_common/test_time.cc | 11 +- test/test_common/test_time.h | 9 +- test/test_common/test_time_system.h | 94 ++++-- test/test_common/utility.cc | 37 +-- test/test_common/utility.h | 9 +- 42 files changed, 472 insertions(+), 561 deletions(-) diff --git a/test/common/formatter/substitution_formatter_fuzz_test.cc b/test/common/formatter/substitution_formatter_fuzz_test.cc index 38356f182a21..afdbb48de80b 100644 --- a/test/common/formatter/substitution_formatter_fuzz_test.cc +++ b/test/common/formatter/substitution_formatter_fuzz_test.cc @@ -19,9 +19,9 @@ DEFINE_PROTO_FUZZER(const test::common::substitution::TestCase& input) { Fuzz::fromHeaders(input.response_headers()); const auto& response_trailers = Fuzz::fromHeaders(input.response_trailers()); - const auto& stream_info = Fuzz::fromStreamInfo(input.stream_info()); + const std::unique_ptr stream_info = Fuzz::fromStreamInfo(input.stream_info()); for (const auto& it : formatters) { - it->format(request_headers, response_headers, response_trailers, stream_info, + it->format(request_headers, response_headers, response_trailers, *stream_info, absl::string_view()); } ENVOY_LOG_MISC(trace, "Success"); diff --git a/test/common/router/header_parser_fuzz_test.cc b/test/common/router/header_parser_fuzz_test.cc index 8acd737fa190..172e67da9415 100644 --- a/test/common/router/header_parser_fuzz_test.cc +++ b/test/common/router/header_parser_fuzz_test.cc @@ -15,8 +15,8 @@ DEFINE_PROTO_FUZZER(const test::common::router::TestCase& input) { Router::HeaderParserPtr parser = Router::HeaderParser::configure(input.headers_to_add(), input.headers_to_remove()); Http::TestRequestHeaderMapImpl header_map; - TestStreamInfo test_stream_info = fromStreamInfo(input.stream_info()); - parser->evaluateHeaders(header_map, test_stream_info); + std::unique_ptr test_stream_info = fromStreamInfo(input.stream_info()); + parser->evaluateHeaders(header_map, *test_stream_info); ENVOY_LOG_MISC(trace, "Success"); } catch (const EnvoyException& e) { ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); diff --git a/test/extensions/filters/common/expr/evaluator_fuzz_test.cc b/test/extensions/filters/common/expr/evaluator_fuzz_test.cc index 95fdc2413c8c..2f29c9f0023c 100644 --- a/test/extensions/filters/common/expr/evaluator_fuzz_test.cc +++ b/test/extensions/filters/common/expr/evaluator_fuzz_test.cc @@ -27,7 +27,7 @@ DEFINE_PROTO_FUZZER(const test::extensions::filters::common::expr::EvaluatorTest // Validate that the input has an expression. TestUtility::validate(input); // Create stream_info to test against, this may catch exceptions from invalid addresses. - stream_info = std::make_unique(Fuzz::fromStreamInfo(input.stream_info())); + stream_info = Fuzz::fromStreamInfo(input.stream_info()); } catch (const EnvoyException& e) { ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); return; diff --git a/test/extensions/filters/http/fault/fault_filter_integration_test.cc b/test/extensions/filters/http/fault/fault_filter_integration_test.cc index a8cd15e97381..957667fad1ae 100644 --- a/test/extensions/filters/http/fault/fault_filter_integration_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_integration_test.cc @@ -125,15 +125,12 @@ TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultConfig) { {":authority", "host"}, {"x-envoy-fault-delay-request", "200"}, {"x-envoy-fault-throughput-response", "1"}}; - const auto current_time = simTime().monotonicTime(); IntegrationStreamDecoderPtr decoder = codec_client_->makeHeaderOnlyRequest(request_headers); + test_server_->waitForCounterEq("http.config_test.fault.delays_injected", 1, + TestUtility::DefaultTimeout, dispatcher_.get()); + simTime().advanceTimeWait(std::chrono::milliseconds(200)); waitForNextUpstreamRequest(); - // At least 200ms of simulated time should have elapsed before we got the upstream request. - EXPECT_LE(std::chrono::milliseconds(200), simTime().monotonicTime() - current_time); - // Active faults gauge is incremented. - EXPECT_EQ(1UL, test_server_->gauge("http.config_test.fault.active_faults")->value()); - // Verify response body throttling. upstream_request_->encodeHeaders(default_response_headers_, false); Buffer::OwnedImpl data(std::string(128, 'a')); @@ -215,6 +212,9 @@ TEST_P(FaultIntegrationTestAllProtocols, HeaderFaultsConfig100PercentageHeaders) {"x-envoy-fault-delay-request-percentage", "100"}, {"x-envoy-fault-throughput-response", "100"}, {"x-envoy-fault-throughput-response-percentage", "100"}}); + test_server_->waitForCounterEq("http.config_test.fault.delays_injected", 1, + TestUtility::DefaultTimeout, dispatcher_.get()); + simTime().advanceTimeWait(std::chrono::milliseconds(100)); waitForNextUpstreamRequest(); upstream_request_->encodeHeaders(default_response_headers_, true); response->waitForEndStream(); diff --git a/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc b/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc index e8c86a00e06d..f0b19bd43806 100644 --- a/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc +++ b/test/extensions/filters/http/ratelimit/ratelimit_integration_test.cc @@ -333,7 +333,7 @@ TEST_P(RatelimitIntegrationTest, ConnectImmediateDisconnect) { initiateClientConnection(); ASSERT_TRUE(fake_upstreams_[1]->waitForHttpConnection(*dispatcher_, fake_ratelimit_connection_)); ASSERT_TRUE(fake_ratelimit_connection_->close()); - ASSERT_TRUE(fake_ratelimit_connection_->waitForDisconnect(true)); + ASSERT_TRUE(fake_ratelimit_connection_->waitForDisconnect()); fake_ratelimit_connection_ = nullptr; // Rate limiter fails open waitForSuccessfulUpstreamResponse(); diff --git a/test/extensions/filters/network/local_ratelimit/local_ratelimit_integration_test.cc b/test/extensions/filters/network/local_ratelimit/local_ratelimit_integration_test.cc index 63b684f49d86..1eaa8c66ff74 100644 --- a/test/extensions/filters/network/local_ratelimit/local_ratelimit_integration_test.cc +++ b/test/extensions/filters/network/local_ratelimit/local_ratelimit_integration_test.cc @@ -40,7 +40,7 @@ name: ratelimit ASSERT_TRUE(fake_upstream_connection->write("world")); tcp_client->waitForData("world"); tcp_client->close(); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); EXPECT_EQ(0, test_server_->counter("local_rate_limit.local_rate_limit_stats.rate_limited")->value()); diff --git a/test/fuzz/utility.h b/test/fuzz/utility.h index 534e5f1f8850..f79aece1041b 100644 --- a/test/fuzz/utility.h +++ b/test/fuzz/utility.h @@ -128,15 +128,15 @@ inline test::fuzz::Headers toHeaders(const Http::HeaderMap& headers) { const std::string TestSubjectPeer = "CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; -inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info) { +inline std::unique_ptr fromStreamInfo(const test::fuzz::StreamInfo& stream_info) { // Set mocks' default string return value to be an empty string. // TODO(asraa): Speed up this function, which is slowed because of the use of mocks. testing::DefaultValue::Set(EMPTY_STRING); - TestStreamInfo test_stream_info; - test_stream_info.metadata_ = stream_info.dynamic_metadata(); + auto test_stream_info = std::make_unique(); + test_stream_info->metadata_ = stream_info.dynamic_metadata(); // Truncate recursive filter metadata fields. // TODO(asraa): Resolve MessageToJsonString failure on recursive filter metadata. - for (auto& pair : *test_stream_info.metadata_.mutable_filter_metadata()) { + for (auto& pair : *test_stream_info->metadata_.mutable_filter_metadata()) { std::string value; pair.second.SerializeToString(&value); pair.second.ParseFromString(value.substr(0, 128)); @@ -147,16 +147,16 @@ inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info) stream_info.start_time() ? 0 : stream_info.start_time() / 1000; - test_stream_info.start_time_ = SystemTime(std::chrono::microseconds(start_time)); + test_stream_info->start_time_ = SystemTime(std::chrono::microseconds(start_time)); if (stream_info.has_response_code()) { - test_stream_info.response_code_ = stream_info.response_code().value(); + test_stream_info->response_code_ = stream_info.response_code().value(); } - test_stream_info.setRequestedServerName(stream_info.requested_server_name()); + test_stream_info->setRequestedServerName(stream_info.requested_server_name()); auto upstream_host = std::make_shared>(); auto upstream_metadata = std::make_shared( replaceInvalidStringValues(stream_info.upstream_metadata())); ON_CALL(*upstream_host, metadata()).WillByDefault(testing::Return(upstream_metadata)); - test_stream_info.upstream_host_ = upstream_host; + test_stream_info->upstream_host_ = upstream_host; auto address = stream_info.has_address() ? Envoy::Network::Address::resolveProtoAddress(stream_info.address()) : Network::Utility::resolveUrl("tcp://10.0.0.1:443"); @@ -164,14 +164,14 @@ inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info) stream_info.has_upstream_local_address() ? Envoy::Network::Address::resolveProtoAddress(stream_info.upstream_local_address()) : Network::Utility::resolveUrl("tcp://10.0.0.1:10000"); - test_stream_info.upstream_local_address_ = upstream_local_address; - test_stream_info.downstream_local_address_ = address; - test_stream_info.downstream_direct_remote_address_ = address; - test_stream_info.downstream_remote_address_ = address; + test_stream_info->upstream_local_address_ = upstream_local_address; + test_stream_info->downstream_local_address_ = address; + test_stream_info->downstream_direct_remote_address_ = address; + test_stream_info->downstream_remote_address_ = address; auto connection_info = std::make_shared>(); ON_CALL(*connection_info, subjectPeerCertificate()) .WillByDefault(testing::ReturnRef(TestSubjectPeer)); - test_stream_info.setDownstreamSslConnection(connection_info); + test_stream_info->setDownstreamSslConnection(connection_info); return test_stream_info; } diff --git a/test/integration/BUILD b/test/integration/BUILD index f54848a6e70c..c789417f3e51 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -896,7 +896,10 @@ envoy_cc_test( envoy_cc_test_library( name = "server_stats_interface", hdrs = ["server_stats.h"], - deps = ["//include/envoy/stats:stats_interface"], + deps = [ + "//include/envoy/event:dispatcher_interface", + "//include/envoy/stats:stats_interface", + ], ) envoy_cc_test( diff --git a/test/integration/autonomous_upstream.cc b/test/integration/autonomous_upstream.cc index 45a467dcf7e7..f673a8e231a6 100644 --- a/test/integration/autonomous_upstream.cc +++ b/test/integration/autonomous_upstream.cc @@ -48,7 +48,7 @@ void AutonomousStream::sendResponse() { int32_t request_body_length = -1; HeaderToInt(EXPECT_REQUEST_SIZE_BYTES, request_body_length, headers); if (request_body_length >= 0) { - EXPECT_EQ(request_body_length, bodyLength()); + EXPECT_EQ(request_body_length, body_.length()); } if (!headers.get_(RESET_AFTER_REQUEST).empty()) { diff --git a/test/integration/autonomous_upstream.h b/test/integration/autonomous_upstream.h index e9d247a4ba95..96c885be345f 100644 --- a/test/integration/autonomous_upstream.h +++ b/test/integration/autonomous_upstream.h @@ -24,11 +24,11 @@ class AutonomousStream : public FakeStream { AutonomousUpstream& upstream, bool allow_incomplete_streams); ~AutonomousStream() override; - void setEndStream(bool set) override; + void setEndStream(bool set) EXCLUSIVE_LOCKS_REQUIRED(lock_) override; private: AutonomousUpstream& upstream_; - void sendResponse(); + void sendResponse() EXCLUSIVE_LOCKS_REQUIRED(lock_); const bool allow_incomplete_streams_{false}; }; diff --git a/test/integration/cluster_filter_integration_test.cc b/test/integration/cluster_filter_integration_test.cc index 4162bc9273cf..7c19415ba1f1 100644 --- a/test/integration/cluster_filter_integration_test.cc +++ b/test/integration/cluster_filter_integration_test.cc @@ -124,7 +124,7 @@ TEST_P(ClusterFilterIntegrationTest, TestClusterFilter) { ASSERT_TRUE(tcp_client->write("", true)); ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); ASSERT_TRUE(fake_upstream_connection->write("", true)); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); tcp_client->waitForDisconnect(); } diff --git a/test/integration/cx_limit_integration_test.cc b/test/integration/cx_limit_integration_test.cc index 126c5c236664..7f0639bf95dd 100644 --- a/test/integration/cx_limit_integration_test.cc +++ b/test/integration/cx_limit_integration_test.cc @@ -47,7 +47,8 @@ class ConnectionLimitIntegrationTest : public testing::TestWithParam= end_time) { - return AssertionFailure() << "Timed out waiting for headers."; - } - time_system_.waitFor(lock_, decoder_event_, 5ms); + absl::MutexLock lock(&lock_); + const auto reached = [this]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return headers_ != nullptr; }; + if (!time_system_.waitFor(lock_, absl::Condition(&reached), timeout)) { + return AssertionFailure() << "Timed out waiting for headers."; } return AssertionSuccess(); } +namespace { +// Perform a wait on a condition while still allowing for periodic client dispatcher runs that +// occur on the current thread. +bool waitForWithDispatcherRun(Event::TestTimeSystem& time_system, absl::Mutex& lock, + const std::function& condition, + Event::Dispatcher& client_dispatcher, milliseconds timeout) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock) { + Event::TestTimeSystem::RealTimeBound bound(timeout); + while (bound.withinBound()) { + // Wake up every 5ms to run the client dispatcher. + if (time_system.waitFor(lock, absl::Condition(&condition), 5ms)) { + return true; + } + + // Run the client dispatcher since we may need to process window updates, etc. + client_dispatcher.run(Event::Dispatcher::RunType::NonBlock); + } + return false; +} +} // namespace + AssertionResult FakeStream::waitForData(Event::Dispatcher& client_dispatcher, uint64_t body_length, milliseconds timeout) { - Thread::LockGuard lock(lock_); - auto start_time = time_system_.monotonicTime(); - while (bodyLength() < body_length) { - if (time_system_.monotonicTime() >= start_time + timeout) { - return AssertionFailure() << "Timed out waiting for data."; - } - time_system_.waitFor(lock_, decoder_event_, 5ms); - if (bodyLength() < body_length) { - // Run the client dispatcher since we may need to process window updates, etc. - client_dispatcher.run(Event::Dispatcher::RunType::NonBlock); - } + absl::MutexLock lock(&lock_); + if (!waitForWithDispatcherRun( + time_system_, lock_, + [this, body_length]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return (body_.length() >= body_length); }, + client_dispatcher, timeout)) { + return AssertionFailure() << "Timed out waiting for data."; } return AssertionSuccess(); } @@ -190,30 +201,20 @@ AssertionResult FakeStream::waitForData(Event::Dispatcher& client_dispatcher, AssertionResult FakeStream::waitForEndStream(Event::Dispatcher& client_dispatcher, milliseconds timeout) { - Thread::LockGuard lock(lock_); - auto start_time = time_system_.monotonicTime(); - while (!end_stream_) { - if (time_system_.monotonicTime() >= start_time + timeout) { - return AssertionFailure() << "Timed out waiting for end of stream."; - } - time_system_.waitFor(lock_, decoder_event_, 5ms); - if (!end_stream_) { - // Run the client dispatcher since we may need to process window updates, etc. - client_dispatcher.run(Event::Dispatcher::RunType::NonBlock); - } + absl::MutexLock lock(&lock_); + if (!waitForWithDispatcherRun( + time_system_, lock_, + [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return end_stream_; }, client_dispatcher, + timeout)) { + return AssertionFailure() << "Timed out waiting for end of stream."; } return AssertionSuccess(); } AssertionResult FakeStream::waitForReset(milliseconds timeout) { - Thread::LockGuard lock(lock_); - auto start_time = time_system_.monotonicTime(); - while (!saw_reset_) { - if (time_system_.monotonicTime() >= start_time + timeout) { - return AssertionFailure() << "Timed out waiting for reset."; - } - // Safe since CondVar::waitFor won't throw. - time_system_.waitFor(lock_, decoder_event_, 5ms); + absl::MutexLock lock(&lock_); + if (!time_system_.waitFor(lock_, absl::Condition(&saw_reset_), timeout)) { + return AssertionFailure() << "Timed out waiting for reset."; } return AssertionSuccess(); } @@ -318,6 +319,7 @@ FakeHttpConnection::FakeHttpConnection( } AssertionResult FakeConnectionBase::close(std::chrono::milliseconds timeout) { + ENVOY_LOG(trace, "FakeConnectionBase close"); if (!shared_connection_.connected()) { return AssertionSuccess(); } @@ -340,88 +342,42 @@ AssertionResult FakeConnectionBase::enableHalfClose(bool enable, } Http::RequestDecoder& FakeHttpConnection::newStream(Http::ResponseEncoder& encoder, bool) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); new_streams_.emplace_back(new FakeStream(*this, encoder, time_system_)); - connection_event_.notifyOne(); return *new_streams_.back(); } -AssertionResult FakeConnectionBase::waitForDisconnect(bool ignore_spurious_events, - milliseconds timeout) { +AssertionResult FakeConnectionBase::waitForDisconnect(milliseconds timeout) { ENVOY_LOG(trace, "FakeConnectionBase waiting for disconnect"); - auto end_time = time_system_.monotonicTime() + timeout; - Thread::LockGuard lock(lock_); - while (shared_connection_.connected()) { - if (time_system_.monotonicTime() >= end_time) { - return AssertionFailure() << "Timed out waiting for disconnect."; - } - Thread::CondVar::WaitStatus status = time_system_.waitFor(lock_, connection_event_, 5ms); - // The default behavior of waitForDisconnect is to assume the test cleanly - // calls waitForData, waitForNewStream, etc. to handle all events on the - // connection. If the caller explicitly notes that other events should be - // ignored, continue looping until a disconnect is detected. Otherwise fall - // through and hit the assert below. - if ((status == Thread::CondVar::WaitStatus::NoTimeout) && !ignore_spurious_events) { - break; - } - } + absl::MutexLock lock(&lock_); + const auto reached = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { + return !shared_connection_.connectedLockHeld(); + }; - if (shared_connection_.connected()) { - return AssertionFailure() << "Expected disconnect, but got a different event."; + if (!time_system_.waitFor(lock_, absl::Condition(&reached), timeout)) { + return AssertionFailure() << "Timed out waiting for disconnect."; } ENVOY_LOG(trace, "FakeConnectionBase done waiting for disconnect"); return AssertionSuccess(); } -AssertionResult FakeConnectionBase::waitForHalfClose(bool ignore_spurious_events, - milliseconds timeout) { - auto end_time = time_system_.monotonicTime() + timeout; - Thread::LockGuard lock(lock_); - while (!half_closed_) { - if (time_system_.monotonicTime() >= end_time) { - return AssertionFailure() << "Timed out waiting for half close."; - } - Thread::CondVar::WaitStatus status = time_system_.waitFor(lock_, connection_event_, 5ms); - // The default behavior of waitForHalfClose is to assume the test cleanly - // calls waitForData, waitForNewStream, etc. to handle all events on the - // connection. If the caller explicitly notes that other events should be - // ignored, continue looping until a disconnect is detected. Otherwise fall - // through and hit the assert below. - if (status == Thread::CondVar::WaitStatus::NoTimeout && !ignore_spurious_events) { - break; - } +AssertionResult FakeConnectionBase::waitForHalfClose(milliseconds timeout) { + absl::MutexLock lock(&lock_); + if (!time_system_.waitFor(lock_, absl::Condition(&half_closed_), timeout)) { + return AssertionFailure() << "Timed out waiting for half close."; } - - return half_closed_ - ? AssertionSuccess() - : (AssertionFailure() << "Expected half close event, but got a different event."); + return AssertionSuccess(); } AssertionResult FakeHttpConnection::waitForNewStream(Event::Dispatcher& client_dispatcher, FakeStreamPtr& stream, - bool ignore_spurious_events, - milliseconds timeout) { - auto end_time = time_system_.monotonicTime() + timeout; - Thread::LockGuard lock(lock_); - while (new_streams_.empty()) { - if (time_system_.monotonicTime() >= end_time) { - return AssertionFailure() << "Timed out waiting for new stream."; - } - Thread::CondVar::WaitStatus status = time_system_.waitFor(lock_, connection_event_, 5ms); - // As with waitForDisconnect, by default, waitForNewStream returns after the next event. - // If the caller explicitly notes other events should be ignored, it will instead actually - // wait for the next new stream, ignoring other events such as onData() - if (status == Thread::CondVar::WaitStatus::NoTimeout && !ignore_spurious_events) { - break; - } - if (new_streams_.empty()) { - // Run the client dispatcher since we may need to process window updates, etc. - client_dispatcher.run(Event::Dispatcher::RunType::NonBlock); - } - } - - if (new_streams_.empty()) { - return AssertionFailure() << "Expected new stream event, but got a different event."; + std::chrono::milliseconds timeout) { + absl::MutexLock lock(&lock_); + if (!waitForWithDispatcherRun( + time_system_, lock_, + [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return !new_streams_.empty(); }, + client_dispatcher, timeout)) { + return AssertionFailure() << "Timed out waiting for new stream."; } stream = std::move(new_streams_.front()); new_streams_.pop_front(); @@ -513,14 +469,13 @@ void FakeUpstream::cleanUp() { bool FakeUpstream::createNetworkFilterChain(Network::Connection& connection, const std::vector&) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); if (read_disable_on_new_connection_) { connection.readDisable(true); } auto connection_wrapper = - std::make_unique(connection, allow_unexpected_disconnects_); + std::make_unique(connection, allow_unexpected_disconnects_); LinkedList::moveIntoListBack(std::move(connection_wrapper), new_connections_); - upstream_event_.notifyOne(); return true; } @@ -537,7 +492,7 @@ void FakeUpstream::threadRoutine() { dispatcher_->run(Event::Dispatcher::RunType::Block); handler_.reset(); { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); new_connections_.clear(); consumed_connections_.clear(); } @@ -548,26 +503,17 @@ AssertionResult FakeUpstream::waitForHttpConnection( uint32_t max_request_headers_kb, uint32_t max_request_headers_count, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action) { - Event::TestTimeSystem& time_system = timeSystem(); - auto end_time = time_system.monotonicTime() + timeout; { - Thread::LockGuard lock(lock_); - while (new_connections_.empty()) { - if (time_system.monotonicTime() >= end_time) { - return AssertionFailure() << "Timed out waiting for new connection."; - } - time_system_.waitFor(lock_, upstream_event_, 5ms); - if (new_connections_.empty()) { - // Run the client dispatcher since we may need to process window updates, etc. - client_dispatcher.run(Event::Dispatcher::RunType::NonBlock); - } + absl::MutexLock lock(&lock_); + if (!waitForWithDispatcherRun( + time_system_, lock_, + [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return !new_connections_.empty(); }, + client_dispatcher, timeout)) { + return AssertionFailure() << "Timed out waiting for new connection."; } - if (new_connections_.empty()) { - return AssertionFailure() << "Got a new connection event, but didn't create a connection."; - } connection = std::make_unique( - *this, consumeConnection(), http_type_, time_system, max_request_headers_kb, + *this, consumeConnection(), http_type_, time_system_, max_request_headers_kb, max_request_headers_count, headers_with_underscores_action); } VERIFY_ASSERTION(connection->initialize()); @@ -584,29 +530,28 @@ FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_dispatcher, if (upstreams.empty()) { return AssertionFailure() << "No upstreams configured."; } - Event::TestTimeSystem& time_system = upstreams[0]->timeSystem(); - auto end_time = time_system.monotonicTime() + timeout; - while (time_system.monotonicTime() < end_time) { + Event::TestTimeSystem::RealTimeBound bound(timeout); + while (bound.withinBound()) { for (auto& it : upstreams) { FakeUpstream& upstream = *it; - Thread::ReleasableLockGuard lock(upstream.lock_); - if (upstream.new_connections_.empty()) { - time_system.waitFor(upstream.lock_, upstream.upstream_event_, 5ms); - } - - if (upstream.new_connections_.empty()) { - // Run the client dispatcher since we may need to process window updates, etc. - client_dispatcher.run(Event::Dispatcher::RunType::NonBlock); - } else { + { + absl::MutexLock lock(&upstream.lock_); + if (!waitForWithDispatcherRun( + upstream.time_system_, upstream.lock_, + [&upstream]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(upstream.lock_) { + return !upstream.new_connections_.empty(); + }, + client_dispatcher, 5ms)) { + continue; + } connection = std::make_unique( upstream, upstream.consumeConnection(), upstream.http_type_, upstream.timeSystem(), Http::DEFAULT_MAX_REQUEST_HEADERS_KB, Http::DEFAULT_MAX_HEADERS_COUNT, envoy::config::core::v3::HttpProtocolOptions::ALLOW); - lock.release(); - VERIFY_ASSERTION(connection->initialize()); - VERIFY_ASSERTION(connection->readDisable(false)); - return AssertionSuccess(); } + VERIFY_ASSERTION(connection->initialize()); + VERIFY_ASSERTION(connection->readDisable(false)); + return AssertionSuccess(); } } return AssertionFailure() << "Timed out waiting for HTTP connection."; @@ -615,14 +560,13 @@ FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_dispatcher, AssertionResult FakeUpstream::waitForRawConnection(FakeRawConnectionPtr& connection, milliseconds timeout) { { - Thread::LockGuard lock(lock_); - if (new_connections_.empty()) { - ENVOY_LOG(debug, "waiting for raw connection"); - time_system_.waitFor(lock_, upstream_event_, - timeout); // Safe since CondVar::waitFor won't throw. - } + absl::MutexLock lock(&lock_); + const auto reached = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { + return !new_connections_.empty(); + }; - if (new_connections_.empty()) { + ENVOY_LOG(debug, "waiting for raw connection"); + if (!time_system_.waitFor(lock_, absl::Condition(&reached), timeout)) { return AssertionFailure() << "Timed out waiting for raw connection"; } connection = std::make_unique(consumeConnection(), timeSystem()); @@ -636,30 +580,30 @@ AssertionResult FakeUpstream::waitForRawConnection(FakeRawConnectionPtr& connect SharedConnectionWrapper& FakeUpstream::consumeConnection() { ASSERT(!new_connections_.empty()); auto* const connection_wrapper = new_connections_.front().get(); - connection_wrapper->set_parented(); + connection_wrapper->setParented(); connection_wrapper->moveBetweenLists(new_connections_, consumed_connections_); - return connection_wrapper->shared_connection(); + return *connection_wrapper; } testing::AssertionResult FakeUpstream::waitForUdpDatagram(Network::UdpRecvData& data_to_fill, std::chrono::milliseconds timeout) { - Thread::LockGuard lock(lock_); - auto end_time = time_system_.monotonicTime() + timeout; - while (received_datagrams_.empty()) { - if (time_system_.monotonicTime() >= end_time) { - return AssertionFailure() << "Timed out waiting for UDP datagram."; - } - time_system_.waitFor(lock_, upstream_event_, 5ms); // Safe since CondVar::waitFor won't throw. + absl::MutexLock lock(&lock_); + const auto reached = [this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { + return !received_datagrams_.empty(); + }; + + if (!time_system_.waitFor(lock_, absl::Condition(&reached), timeout)) { + return AssertionFailure() << "Timed out waiting for UDP datagram."; } + data_to_fill = std::move(received_datagrams_.front()); received_datagrams_.pop_front(); return AssertionSuccess(); } void FakeUpstream::onRecvDatagram(Network::UdpRecvData& data) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); received_datagrams_.emplace_back(std::move(data)); - upstream_event_.notifyOne(); } void FakeUpstream::sendUdpDatagram(const std::string& buffer, @@ -673,14 +617,13 @@ void FakeUpstream::sendUdpDatagram(const std::string& buffer, AssertionResult FakeRawConnection::waitForData(uint64_t num_bytes, std::string* data, milliseconds timeout) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); + const auto reached = [this, num_bytes]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { + return data_.size() == num_bytes; + }; ENVOY_LOG(debug, "waiting for {} bytes of data", num_bytes); - auto end_time = time_system_.monotonicTime() + timeout; - while (data_.size() != num_bytes) { - if (time_system_.monotonicTime() >= end_time) { - return AssertionFailure() << "Timed out waiting for data."; - } - time_system_.waitFor(lock_, connection_event_, 5ms); // Safe since CondVar::waitFor won't throw. + if (!time_system_.waitFor(lock_, absl::Condition(&reached), timeout)) { + return AssertionFailure() << "Timed out waiting for data."; } if (data != nullptr) { *data = data_; @@ -691,14 +634,12 @@ AssertionResult FakeRawConnection::waitForData(uint64_t num_bytes, std::string* AssertionResult FakeRawConnection::waitForData(const std::function& data_validator, std::string* data, milliseconds timeout) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); + const auto reached = [this, &data_validator]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { return data_validator(data_); }; ENVOY_LOG(debug, "waiting for data"); - auto end_time = time_system_.monotonicTime() + timeout; - while (!data_validator(data_)) { - if (time_system_.monotonicTime() >= end_time) { - return AssertionFailure() << "Timed out waiting for data."; - } - time_system_.waitFor(lock_, connection_event_, 5ms); // Safe since CondVar::waitFor won't throw. + if (!time_system_.waitFor(lock_, absl::Condition(&reached), timeout)) { + return AssertionFailure() << "Timed out waiting for data."; } if (data != nullptr) { *data = data_; @@ -718,12 +659,11 @@ AssertionResult FakeRawConnection::write(const std::string& data, bool end_strea Network::FilterStatus FakeRawConnection::ReadFilter::onData(Buffer::Instance& data, bool end_stream) { - Thread::LockGuard lock(parent_.lock_); + absl::MutexLock lock(&parent_.lock_); ENVOY_LOG(debug, "got {} bytes, end_stream {}", data.length(), end_stream); parent_.data_.append(data.toString()); parent_.half_closed_ = end_stream; data.drain(data.length()); - parent_.connection_event_.notifyOne(); return Network::FilterStatus::StopIteration; } } // namespace Envoy diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index c9dd430d2440..e067d09024aa 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -41,6 +41,8 @@ #include "test/test_common/test_time_system.h" #include "test/test_common/utility.h" +// TODO(mattklein123): A lot of code should be moved from this header file into the cc file. + namespace Envoy { class FakeHttpConnection; @@ -56,10 +58,16 @@ class FakeStream : public Http::RequestDecoder, FakeStream(FakeHttpConnection& parent, Http::ResponseEncoder& encoder, Event::TestTimeSystem& time_system); - uint64_t bodyLength() { return body_.length(); } - Buffer::Instance& body() { return body_; } + uint64_t bodyLength() { + absl::MutexLock lock(&lock_); + return body_.length(); + } + Buffer::Instance& body() { + absl::MutexLock lock(&lock_); + return body_; + } bool complete() { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); return end_stream_; } @@ -76,7 +84,10 @@ class FakeStream : public Http::RequestDecoder, void encodeResetStream(); void encodeMetadata(const Http::MetadataMapVector& metadata_map_vector); void readDisable(bool disable); - const Http::RequestHeaderMap& headers() { return *headers_; } + const Http::RequestHeaderMap& headers() { + absl::MutexLock lock(&lock_); + return *headers_; + } void setAddServedByHeader(bool add_header) { add_served_by_header_ = add_header; } const Http::RequestTrailerMapPtr& trailers() { return trailers_; } bool receivedData() { return received_data_; } @@ -88,8 +99,12 @@ class FakeStream : public Http::RequestDecoder, const std::function& /*modify_headers*/, const absl::optional grpc_status, absl::string_view /*details*/) override { - const bool is_head_request = - headers_ != nullptr && headers_->getMethodValue() == Http::Headers::get().MethodValues.Head; + bool is_head_request; + { + absl::MutexLock lock(&lock_); + is_head_request = headers_ != nullptr && + headers_->getMethodValue() == Http::Headers::get().MethodValues.Head; + } Http::Utility::sendLocalReply( false, Http::Utility::EncodeFunctions( @@ -150,7 +165,7 @@ class FakeStream : public Http::RequestDecoder, ABSL_MUST_USE_RESULT testing::AssertionResult waitForGrpcMessage(Event::Dispatcher& client_dispatcher, T& message, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) { - auto end_time = timeSystem().monotonicTime() + timeout; + Event::TestTimeSystem::RealTimeBound bound(timeout); ENVOY_LOG(debug, "Waiting for gRPC message..."); if (!decoded_grpc_frames_.empty()) { decodeGrpcFrame(message); @@ -160,23 +175,21 @@ class FakeStream : public Http::RequestDecoder, return testing::AssertionFailure() << "Timed out waiting for start of gRPC message."; } { - Thread::LockGuard lock(lock_); - if (!grpc_decoder_.decode(body(), decoded_grpc_frames_)) { + absl::MutexLock lock(&lock_); + if (!grpc_decoder_.decode(body_, decoded_grpc_frames_)) { return testing::AssertionFailure() - << "Couldn't decode gRPC data frame: " << body().toString(); + << "Couldn't decode gRPC data frame: " << body_.toString(); } } if (decoded_grpc_frames_.empty()) { - timeout = std::chrono::duration_cast(end_time - - timeSystem().monotonicTime()); - if (!waitForData(client_dispatcher, grpc_decoder_.length(), timeout)) { + if (!waitForData(client_dispatcher, grpc_decoder_.length(), bound.timeLeft())) { return testing::AssertionFailure() << "Timed out waiting for end of gRPC message."; } { - Thread::LockGuard lock(lock_); - if (!grpc_decoder_.decode(body(), decoded_grpc_frames_)) { + absl::MutexLock lock(&lock_); + if (!grpc_decoder_.decode(body_, decoded_grpc_frames_)) { return testing::AssertionFailure() - << "Couldn't decode gRPC data frame: " << body().toString(); + << "Couldn't decode gRPC data frame: " << body_.toString(); } } } @@ -199,7 +212,7 @@ class FakeStream : public Http::RequestDecoder, void onAboveWriteBufferHighWatermark() override {} void onBelowWriteBufferLowWatermark() override {} - virtual void setEndStream(bool end) { end_stream_ = end; } + virtual void setEndStream(bool end) EXCLUSIVE_LOCKS_REQUIRED(lock_) { end_stream_ = end; } Event::TestTimeSystem& timeSystem() { return time_system_; } @@ -209,17 +222,16 @@ class FakeStream : public Http::RequestDecoder, } protected: - Http::RequestHeaderMapPtr headers_; + absl::Mutex lock_; + Http::RequestHeaderMapPtr headers_ ABSL_GUARDED_BY(lock_); + Buffer::OwnedImpl body_ ABSL_GUARDED_BY(lock_); private: FakeHttpConnection& parent_; Http::ResponseEncoder& encoder_; - Thread::MutexBasicLockable lock_; - Thread::CondVar decoder_event_; - Http::RequestTrailerMapPtr trailers_; - bool end_stream_{}; - Buffer::OwnedImpl body_; - bool saw_reset_{}; + Http::RequestTrailerMapPtr trailers_ ABSL_GUARDED_BY(lock_); + bool end_stream_ ABSL_GUARDED_BY(lock_){}; + bool saw_reset_ ABSL_GUARDED_BY(lock_){}; Grpc::Decoder grpc_decoder_; std::vector decoded_grpc_frames_; bool add_served_by_header_{}; @@ -239,33 +251,43 @@ using FakeStreamPtr = std::unique_ptr; // SharedConnectionWrapper that lives from when the Connection is added to the accepted connection // queue and then through the lifetime of the Fake{Raw,Http}Connection that manages the Connection // through active use. -class SharedConnectionWrapper : public Network::ConnectionCallbacks { +class SharedConnectionWrapper : public Network::ConnectionCallbacks, + public LinkedObject { public: using DisconnectCallback = std::function; SharedConnectionWrapper(Network::Connection& connection, bool allow_unexpected_disconnects) : connection_(connection), allow_unexpected_disconnects_(allow_unexpected_disconnects) { connection_.addConnectionCallbacks(*this); + addDisconnectCallback([this]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) { + RELEASE_ASSERT(parented_ || allow_unexpected_disconnects_, + "An queued upstream connection was torn down without being associated " + "with a fake connection. Either manage the connection via " + "waitForRawConnection() or waitForHttpConnection(), or " + "set_allow_unexpected_disconnects(true).\n See " + "https://github.com/envoyproxy/envoy/blob/master/test/integration/README.md#" + "unparented-upstream-connections"); + }); } Common::CallbackHandle* addDisconnectCallback(DisconnectCallback callback) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); return disconnect_callback_manager_.add(callback); } // Avoid directly removing by caller, since CallbackManager is not thread safe. void removeDisconnectCallback(Common::CallbackHandle* handle) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); handle->remove(); } // Network::ConnectionCallbacks void onEvent(Network::ConnectionEvent event) override { // Throughout this entire function, we know that the connection_ cannot disappear, since this - // callback is invoked prior to connection_ deferred delete. We also know by locking below, that - // elsewhere where we also hold lock_, that the connection cannot disappear inside the locked - // scope. - Thread::LockGuard lock(lock_); + // callback is invoked prior to connection_ deferred delete. We also know by locking below, + // that elsewhere where we also hold lock_, that the connection cannot disappear inside the + // locked scope. + absl::MutexLock lock(&lock_); if (event == Network::ConnectionEvent::RemoteClose || event == Network::ConnectionEvent::LocalClose) { disconnected_ = true; @@ -277,7 +299,14 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks { void onBelowWriteBufferLowWatermark() override {} bool connected() { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); + return connectedLockHeld(); + } + + bool connectedLockHeld() { + lock_.AssertReaderHeld(); // TODO(mattklein123): This can't be annotated because the lock + // is acquired via the base connection reference. Fix this to + // remove the reference. return !disconnected_; } @@ -295,28 +324,28 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks { testing::AssertionResult executeOnDispatcher(std::function f, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout) { - Thread::LockGuard lock(lock_); + absl::MutexLock lock(&lock_); if (disconnected_) { return testing::AssertionSuccess(); } - Thread::CondVar callback_ready_event; - std::atomic unexpected_disconnect = false; + bool callback_ready_event = false; + bool unexpected_disconnect = false; connection_.dispatcher().post( - [this, f, &callback_ready_event, &unexpected_disconnect]() -> void { + [this, f, &lock = lock_, &callback_ready_event, &unexpected_disconnect]() -> void { // The use of connected() here, vs. !disconnected_, is because we want to use the lock_ - // acquisition to briefly serialize. This avoids us entering this completion and issuing a - // notifyOne() until the wait() is ready to receive it below. + // acquisition to briefly serialize. This avoids us entering this completion and issuing + // a notifyOne() until the wait() is ready to receive it below. if (connected()) { f(connection_); } else { unexpected_disconnect = true; } - callback_ready_event.notifyOne(); + absl::MutexLock lock_guard(&lock); + callback_ready_event = true; }); Event::TestTimeSystem& time_system = dynamic_cast(connection_.dispatcher().timeSource()); - Thread::CondVar::WaitStatus status = time_system.waitFor(lock_, callback_ready_event, timeout); - if (status == Thread::CondVar::WaitStatus::Timeout) { + if (!time_system.waitFor(lock_, absl::Condition(&callback_ready_event), timeout)) { return testing::AssertionFailure() << "Timed out while executing on dispatcher."; } if (unexpected_disconnect && !allow_unexpected_disconnects_) { @@ -330,69 +359,30 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks { return testing::AssertionSuccess(); } + absl::Mutex& lock() { return lock_; } + + void setParented() { + absl::MutexLock lock(&lock_); + parented_ = true; + } + private: Network::Connection& connection_; - Thread::MutexBasicLockable lock_; + absl::Mutex lock_; Common::CallbackManager<> disconnect_callback_manager_ ABSL_GUARDED_BY(lock_); + bool parented_ ABSL_GUARDED_BY(lock_){}; bool disconnected_ ABSL_GUARDED_BY(lock_){}; const bool allow_unexpected_disconnects_; }; using SharedConnectionWrapperPtr = std::unique_ptr; -class QueuedConnectionWrapper; -using QueuedConnectionWrapperPtr = std::unique_ptr; - -/** - * Wraps a raw Network::Connection in a safe way, such that the connection can - * be placed in a queue for an arbitrary amount of time. It handles disconnects - * that take place in the queued state by failing the test. Once a - * QueuedConnectionWrapper object is instantiated by FakeHttpConnection or - * FakeRawConnection, it no longer plays a role. - * TODO(htuch): We can simplify the storage lifetime by destructing if/when - * removeConnectionCallbacks is added. - */ -class QueuedConnectionWrapper : public LinkedObject { -public: - QueuedConnectionWrapper(Network::Connection& connection, bool allow_unexpected_disconnects) - : shared_connection_(connection, allow_unexpected_disconnects), parented_(false), - allow_unexpected_disconnects_(allow_unexpected_disconnects) { - shared_connection_.addDisconnectCallback([this] { - Thread::LockGuard lock(lock_); - RELEASE_ASSERT(parented_ || allow_unexpected_disconnects_, - "An queued upstream connection was torn down without being associated " - "with a fake connection. Either manage the connection via " - "waitForRawConnection() or waitForHttpConnection(), or " - "set_allow_unexpected_disconnects(true).\n See " - "https://github.com/envoyproxy/envoy/blob/master/test/integration/README.md#" - "unparented-upstream-connections"); - }); - } - - void set_parented() { - Thread::LockGuard lock(lock_); - parented_ = true; - } - - SharedConnectionWrapper& shared_connection() { return shared_connection_; } - -private: - SharedConnectionWrapper shared_connection_; - Thread::MutexBasicLockable lock_; - bool parented_ ABSL_GUARDED_BY(lock_); - const bool allow_unexpected_disconnects_; -}; - /** * Base class for both fake raw connections and fake HTTP connections. */ class FakeConnectionBase : public Logger::Loggable { public: - virtual ~FakeConnectionBase() { - ASSERT(initialized_); - ASSERT(disconnect_callback_handle_ != nullptr); - shared_connection_.removeDisconnectCallback(disconnect_callback_handle_); - } + virtual ~FakeConnectionBase() { ASSERT(initialized_); } ABSL_MUST_USE_RESULT testing::AssertionResult close(std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); @@ -401,44 +391,35 @@ class FakeConnectionBase : public Logger::Loggable { testing::AssertionResult readDisable(bool disable, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); - // By default waitForDisconnect and waitForHalfClose assume the next event is - // a disconnect and return an AssertionFailure if an unexpected event occurs. - // If a caller truly wishes to wait until disconnect, set - // ignore_spurious_events = true. ABSL_MUST_USE_RESULT testing::AssertionResult - waitForDisconnect(bool ignore_spurious_events = false, - std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); + waitForDisconnect(std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); ABSL_MUST_USE_RESULT testing::AssertionResult - waitForHalfClose(bool ignore_spurious_events = false, - std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); + waitForHalfClose(std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); ABSL_MUST_USE_RESULT virtual testing::AssertionResult initialize() { initialized_ = true; - disconnect_callback_handle_ = - shared_connection_.addDisconnectCallback([this] { connection_event_.notifyOne(); }); return testing::AssertionSuccess(); } ABSL_MUST_USE_RESULT testing::AssertionResult enableHalfClose(bool enabled, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); - SharedConnectionWrapper& shared_connection() { return shared_connection_; } // The same caveats apply here as in SharedConnectionWrapper::connection(). Network::Connection& connection() const { return shared_connection_.connection(); } bool connected() const { return shared_connection_.connected(); } protected: FakeConnectionBase(SharedConnectionWrapper& shared_connection, Event::TestTimeSystem& time_system) - : shared_connection_(shared_connection), time_system_(time_system) {} + : shared_connection_(shared_connection), lock_(shared_connection.lock()), + time_system_(time_system) {} - Common::CallbackHandle* disconnect_callback_handle_; SharedConnectionWrapper& shared_connection_; bool initialized_{}; - Thread::CondVar connection_event_; - Thread::MutexBasicLockable lock_; + absl::Mutex& lock_; // TODO(mattklein123): Use the shared connection lock and figure out better + // guarded by annotations. bool half_closed_ ABSL_GUARDED_BY(lock_){}; Event::TestTimeSystem& time_system_; }; @@ -456,14 +437,9 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action); - // By default waitForNewStream assumes the next event is a new stream and - // returns AssertionFailure if an unexpected event occurs. If a caller truly - // wishes to wait for a new stream, set ignore_spurious_events = true. Returns - // the new stream via the stream argument. ABSL_MUST_USE_RESULT testing::AssertionResult waitForNewStream(Event::Dispatcher& client_dispatcher, FakeStreamPtr& stream, - bool ignore_spurious_events = false, std::chrono::milliseconds timeout = TestUtility::DefaultTimeout); // Http::ServerConnectionCallbacks @@ -498,7 +474,7 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo }; Http::ServerConnectionPtr codec_; - std::list new_streams_; + std::list new_streams_ ABSL_GUARDED_BY(lock_); }; using FakeHttpConnectionPtr = std::unique_ptr; @@ -563,7 +539,7 @@ class FakeRawConnection : public FakeConnectionBase { FakeRawConnection& parent_; }; - std::string data_; + std::string data_ ABSL_GUARDED_BY(lock_); }; using FakeRawConnectionPtr = std::unique_ptr; @@ -756,18 +732,17 @@ class FakeUpstream : Logger::Loggable, ConditionalInitializer server_initialized_; // Guards any objects which can be altered both in the upstream thread and the // main test thread. - Thread::MutexBasicLockable lock_; + absl::Mutex lock_; Thread::ThreadPtr thread_; - Thread::CondVar upstream_event_; Api::ApiPtr api_; Event::TestTimeSystem& time_system_; Event::DispatcherPtr dispatcher_; Network::ConnectionHandlerPtr handler_; - std::list new_connections_ ABSL_GUARDED_BY(lock_); + std::list new_connections_ ABSL_GUARDED_BY(lock_); // When a QueuedConnectionWrapper is popped from new_connections_, ownership is transferred to // consumed_connections_. This allows later the Connection destruction (when the FakeUpstream is // deleted) on the same thread that allocated the connection. - std::list consumed_connections_ ABSL_GUARDED_BY(lock_); + std::list consumed_connections_ ABSL_GUARDED_BY(lock_); bool allow_unexpected_disconnects_; bool read_disable_on_new_connection_; const bool enable_half_close_; diff --git a/test/integration/filter_manager_integration_test.cc b/test/integration/filter_manager_integration_test.cc index 0d1c55afa896..d6d43ba0a66a 100644 --- a/test/integration/filter_manager_integration_test.cc +++ b/test/integration/filter_manager_integration_test.cc @@ -515,7 +515,7 @@ TEST_P(InjectDataWithTcpProxyFilterIntegrationTest, UsageOfInjectDataMethodsShou ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); ASSERT_TRUE(fake_upstream_connection->write("there!", true)); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); tcp_client->waitForData("there!"); tcp_client->waitForDisconnect(); diff --git a/test/integration/h1_fuzz.cc b/test/integration/h1_fuzz.cc index 3fe7886bd970..5067240254e5 100644 --- a/test/integration/h1_fuzz.cc +++ b/test/integration/h1_fuzz.cc @@ -69,7 +69,7 @@ void H1FuzzIntegrationTest::replay(const test::integration::CaptureFuzzTestCase& AssertionResult result = fake_upstream_connection->close(); RELEASE_ASSERT(result, result.message()); } - AssertionResult result = fake_upstream_connection->waitForDisconnect(true); + AssertionResult result = fake_upstream_connection->waitForDisconnect(); RELEASE_ASSERT(result, result.message()); } tcp_client->close(); diff --git a/test/integration/h2_fuzz.cc b/test/integration/h2_fuzz.cc index c0eeae08152e..a7c7e3195c33 100644 --- a/test/integration/h2_fuzz.cc +++ b/test/integration/h2_fuzz.cc @@ -244,7 +244,7 @@ void H2FuzzIntegrationTest::replay(const test::integration::H2CaptureFuzzTestCas AssertionResult result = fake_upstream_connection->close(); RELEASE_ASSERT(result, result.message()); } - AssertionResult result = fake_upstream_connection->waitForDisconnect(true); + AssertionResult result = fake_upstream_connection->waitForDisconnect(); RELEASE_ASSERT(result, result.message()); } if (tcp_client->connected()) { diff --git a/test/integration/hds_integration_test.cc b/test/integration/hds_integration_test.cc index 1c950e2499b1..2073cb2a6825 100644 --- a/test/integration/hds_integration_test.cc +++ b/test/integration/hds_integration_test.cc @@ -317,7 +317,7 @@ TEST_P(HdsIntegrationTest, SingleEndpointTimeoutHttp) { ASSERT_TRUE(host_upstream_->waitForRawConnection(host_fake_raw_connection_)); // Endpoint doesn't respond to the health check - ASSERT_TRUE(host_fake_raw_connection_->waitForDisconnect(true)); + ASSERT_TRUE(host_fake_raw_connection_->waitForDisconnect()); // Receive updates until the one we expect arrives waitForEndpointHealthResponse(envoy::config::core::v3::TIMEOUT); @@ -391,7 +391,7 @@ TEST_P(HdsIntegrationTest, SingleEndpointTimeoutTcp) { ASSERT_TRUE(host_upstream_->waitForRawConnection(host_fake_raw_connection_)); // No response from the endpoint - ASSERT_TRUE(host_fake_raw_connection_->waitForDisconnect(true)); + ASSERT_TRUE(host_fake_raw_connection_->waitForDisconnect()); // Receive updates until the one we expect arrives waitForEndpointHealthResponse(envoy::config::core::v3::TIMEOUT); diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 2cc24c148bc7..19037786f98d 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1318,8 +1318,7 @@ void Http2RingHashIntegrationTest::sendMultipleRequests( // As data and streams are interwoven, make sure waitForNewStream() // ignores incoming data and waits for actual stream establishment. upstream_requests.emplace_back(); - ASSERT_TRUE( - fake_upstream_connection->waitForNewStream(*dispatcher_, upstream_requests.back(), true)); + ASSERT_TRUE(fake_upstream_connection->waitForNewStream(*dispatcher_, upstream_requests.back())); upstream_requests.back()->setAddServedByHeader(true); fake_upstream_connections_.push_back(std::move(fake_upstream_connection)); } diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 56d738e3b9ce..9bc332fe9f05 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -364,8 +364,7 @@ HttpIntegrationTest::waitForNextUpstreamRequest(const std::vector& ups if (!fake_upstream_connection_) { AssertionResult result = AssertionFailure(); int upstream_index = 0; - Event::TestTimeSystem& time_system = timeSystem(); - auto end_time = time_system.monotonicTime() + connection_wait_timeout; + Event::TestTimeSystem::RealTimeBound bound(connection_wait_timeout); // Loop over the upstreams until the call times out or an upstream request is received. while (!result) { upstream_index = upstream_index % upstream_indices.size(); @@ -375,7 +374,7 @@ HttpIntegrationTest::waitForNextUpstreamRequest(const std::vector& ups if (result) { upstream_with_request = upstream_index; break; - } else if (time_system.monotonicTime() >= end_time) { + } else if (!bound.withinBound()) { result = (AssertionFailure() << "Timed out waiting for new connection."); break; } diff --git a/test/integration/http_timeout_integration_test.cc b/test/integration/http_timeout_integration_test.cc index 4592533656f0..182c273918e1 100644 --- a/test/integration/http_timeout_integration_test.cc +++ b/test/integration/http_timeout_integration_test.cc @@ -243,10 +243,13 @@ TEST_P(HttpTimeoutIntegrationTest, PerTryTimeout) { ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); - // Trigger per try timeout (but not global timeout). + // Trigger per try timeout (but not global timeout) and wait for reset. timeSystem().advanceTimeWait(std::chrono::milliseconds(400)); + ASSERT_TRUE(upstream_request_->waitForReset()); - // Wait for a second request to be sent upstream + // Wait for a second request to be sent upstream. Max retry backoff is 25ms so advance time that + // much. + timeSystem().advanceTimeWait(std::chrono::milliseconds(25)); ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); @@ -290,10 +293,13 @@ TEST_P(HttpTimeoutIntegrationTest, PerTryTimeoutWithoutGlobalTimeout) { ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); - // Trigger per try timeout. + // Trigger per try timeout (but not global timeout) and wait for reset. timeSystem().advanceTimeWait(std::chrono::milliseconds(5)); + ASSERT_TRUE(upstream_request_->waitForReset()); - // Wait for a second request to be sent upstream + // Wait for a second request to be sent upstream. Max retry backoff is 25ms so advance time that + // much. + timeSystem().advanceTimeWait(std::chrono::milliseconds(25)); ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 979e3fd8c47b..f0f882499b6d 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -149,10 +149,10 @@ void IntegrationStreamDecoder::onResetStream(Http::StreamResetReason reason, abs } IntegrationTcpClient::IntegrationTcpClient( - Event::Dispatcher& dispatcher, Event::TestTimeSystem& time_system, MockBufferFactory& factory, - uint32_t port, Network::Address::IpVersion version, bool enable_half_close, + Event::Dispatcher& dispatcher, MockBufferFactory& factory, uint32_t port, + Network::Address::IpVersion version, bool enable_half_close, const Network::ConnectionSocket::OptionsSharedPtr& options) - : time_system_(time_system), payload_reader_(new WaitForPayloadReader(dispatcher)), + : payload_reader_(new WaitForPayloadReader(dispatcher)), callbacks_(new ConnectionCallbacks(*this)) { EXPECT_CALL(factory, create_(_, _, _)) .WillOnce(Invoke([&](std::function below_low, std::function above_high, @@ -225,7 +225,7 @@ void IntegrationTcpClient::readDisable(bool disabled) { connection_->readDisable AssertionResult IntegrationTcpClient::write(const std::string& data, bool end_stream, bool verify, std::chrono::milliseconds timeout) { - auto end_time = time_system_.monotonicTime() + timeout; + Event::TestTimeSystem::RealTimeBound bound(timeout); Buffer::OwnedImpl buffer(data); if (verify) { EXPECT_CALL(*client_write_buffer_, move(_)); @@ -242,9 +242,9 @@ AssertionResult IntegrationTcpClient::write(const std::string& data, bool end_st if (client_write_buffer_->bytes_written() == bytes_expected || disconnected_) { break; } - } while (time_system_.monotonicTime() < end_time); + } while (bound.withinBound()); - if (time_system_.monotonicTime() >= end_time) { + if (!bound.withinBound()) { return AssertionFailure() << "Timed out completing write"; } else if (verify && (disconnected_ || client_write_buffer_->bytes_written() != bytes_expected)) { return AssertionFailure() @@ -279,7 +279,7 @@ BaseIntegrationTest::BaseIntegrationTest(const InstanceConstSharedPtrFn& upstrea // notification and clear the pool connection if necessary. A real fix would require adding fairly // complex test hooks to the server and/or spin waiting on stats, neither of which I think are // necessary right now. - timeSystem().advanceTimeWait(std::chrono::milliseconds(10)); + timeSystem().realSleepDoNotUseWithoutScrutiny(std::chrono::milliseconds(10)); ON_CALL(*mock_buffer_factory_, create_(_, _, _)) .WillByDefault(Invoke([](std::function below_low, std::function above_high, std::function above_overflow) -> Buffer::Instance* { @@ -412,8 +412,8 @@ void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) IntegrationTcpClientPtr BaseIntegrationTest::makeTcpConnection(uint32_t port, const Network::ConnectionSocket::OptionsSharedPtr& options) { - return std::make_unique(*dispatcher_, time_system_, *mock_buffer_factory_, - port, version_, enable_half_close_, options); + return std::make_unique(*dispatcher_, *mock_buffer_factory_, port, version_, + enable_half_close_, options); } void BaseIntegrationTest::registerPort(const std::string& key, uint32_t port) { @@ -485,7 +485,7 @@ void BaseIntegrationTest::createGeneratedApiTestServer( // Wait for listeners to be created before invoking registerTestServerPorts() below, as that // needs to know about the bound listener ports. - auto end_time = time_system_.monotonicTime() + TestUtility::DefaultTimeout; + Event::TestTimeSystem::RealTimeBound bound(TestUtility::DefaultTimeout); const char* success = "listener_manager.listener_create_success"; const char* rejected = "listener_manager.lds.update_rejected"; for (Stats::CounterSharedPtr success_counter = test_server_->counter(success), @@ -496,7 +496,7 @@ void BaseIntegrationTest::createGeneratedApiTestServer( (!allow_lds_rejection || rejected_counter == nullptr || rejected_counter->value() == 0); success_counter = test_server_->counter(success), rejected_counter = test_server_->counter(rejected)) { - if (time_system_.monotonicTime() >= end_time) { + if (!bound.withinBound()) { RELEASE_ASSERT(0, "Timed out waiting for listeners."); } if (!allow_lds_rejection) { @@ -504,7 +504,8 @@ void BaseIntegrationTest::createGeneratedApiTestServer( absl::StrCat("Lds update failed. Details\n", getListenerDetails(test_server_->server()))); } - time_system_.advanceTimeWait(std::chrono::milliseconds(10)); + // TODO(mattklein123): Switch to events and waitFor(). + time_system_.realSleepDoNotUseWithoutScrutiny(std::chrono::milliseconds(10)); } registerTestServerPorts(port_names); @@ -698,15 +699,16 @@ AssertionResult compareSets(const std::set& set1, const std::set; */ class IntegrationTcpClient { public: - IntegrationTcpClient(Event::Dispatcher& dispatcher, Event::TestTimeSystem& time_system, - MockBufferFactory& factory, uint32_t port, + IntegrationTcpClient(Event::Dispatcher& dispatcher, MockBufferFactory& factory, uint32_t port, Network::Address::IpVersion version, bool enable_half_close, const Network::ConnectionSocket::OptionsSharedPtr& options); @@ -131,7 +130,6 @@ class IntegrationTcpClient { IntegrationTcpClient& parent_; }; - Event::TestTimeSystem& time_system_; std::shared_ptr payload_reader_; std::shared_ptr callbacks_; Network::ClientConnectionPtr connection_; diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index 9efe9bab87b8..6c526fbb7425 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -1432,7 +1432,7 @@ TEST_P(IntegrationTest, Response204WithBody) { // should still see a response. upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "204"}}, false); upstream_request_->encodeData(512, true); - ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); response->waitForEndStream(); diff --git a/test/integration/load_stats_integration_test.cc b/test/integration/load_stats_integration_test.cc index ef6402e71403..3567488a9e4e 100644 --- a/test/integration/load_stats_integration_test.cc +++ b/test/integration/load_stats_integration_test.cc @@ -242,7 +242,7 @@ class LoadStatsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationPara waitForLoadStatsRequest(const std::vector& expected_locality_stats, uint64_t dropped = 0) { - auto end_time = timeSystem().monotonicTime() + TestUtility::DefaultTimeout; + Event::TestTimeSystem::RealTimeBound bound(TestUtility::DefaultTimeout); Protobuf::RepeatedPtrField expected_cluster_stats; if (!expected_locality_stats.empty() || dropped != 0) { auto* cluster_stats = expected_cluster_stats.Add(); @@ -289,7 +289,7 @@ class LoadStatsIntegrationTest : public Grpc::VersionedGrpcClientIntegrationPara "StreamLoadStats", apiVersion()), loadstats_stream_->headers().getPathValue()); EXPECT_EQ("application/grpc", loadstats_stream_->headers().getContentTypeValue()); - if (timeSystem().monotonicTime() >= end_time) { + if (!bound.withinBound()) { return TestUtility::assertRepeatedPtrFieldEqual(expected_cluster_stats, loadstats_request.cluster_stats(), true); } diff --git a/test/integration/redirect_integration_test.cc b/test/integration/redirect_integration_test.cc index 0b03e662e9e8..7deca7a13c15 100644 --- a/test/integration/redirect_integration_test.cc +++ b/test/integration/redirect_integration_test.cc @@ -48,8 +48,8 @@ class RedirectIntegrationTest : public HttpProtocolIntegrationTest { FakeStreamPtr new_stream = nullptr; auto wait_new_stream_fn = [this, &new_stream](FakeHttpConnectionPtr& connection) -> AssertionResult { - AssertionResult result = connection->waitForNewStream(*dispatcher_, new_stream, false, - std::chrono::milliseconds(50)); + AssertionResult result = + connection->waitForNewStream(*dispatcher_, new_stream, std::chrono::milliseconds(50)); if (result) { ASSERT(new_stream); } diff --git a/test/integration/server.h b/test/integration/server.h index 4dc9a3ee21ea..65cad7f5ea88 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -421,10 +421,11 @@ class IntegrationTestServer : public Logger::Loggable, Server::FieldValidationConfig validation_config, uint32_t concurrency, std::chrono::seconds drain_time, Server::DrainStrategy drain_strategy); - void - waitForCounterEq(const std::string& name, uint64_t value, - std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) override { - ASSERT_TRUE(TestUtility::waitForCounterGe(statStore(), name, value, time_system_, timeout)); + void waitForCounterEq(const std::string& name, uint64_t value, + std::chrono::milliseconds timeout = std::chrono::milliseconds::zero(), + Event::Dispatcher* dispatcher = nullptr) override { + ASSERT_TRUE( + TestUtility::waitForCounterEq(statStore(), name, value, time_system_, timeout, dispatcher)); } void diff --git a/test/integration/server_stats.h b/test/integration/server_stats.h index c3ab300f0505..6c169ed68dd4 100644 --- a/test/integration/server_stats.h +++ b/test/integration/server_stats.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/event/dispatcher.h" #include "envoy/stats/stats.h" namespace Envoy { @@ -14,10 +15,12 @@ class IntegrationTestServerStats { * @param name counter name. * @param value target value. * @param timeout amount of time to wait before asserting false, or 0 for no timeout. + * @param dispatcher the dispatcher to run non-blocking periodically during the wait. */ virtual void waitForCounterEq(const std::string& name, uint64_t value, - std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()) PURE; + std::chrono::milliseconds timeout = std::chrono::milliseconds::zero(), + Event::Dispatcher* dispatcher = nullptr) PURE; /** * Wait for a counter to >= a given value. diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index 39d6a4779740..5533308b5dc4 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -121,7 +121,7 @@ TEST_P(TcpProxyIntegrationTest, TcpProxyDownstreamDisconnect) { ASSERT_TRUE(fake_upstream_connection->waitForData(10)); ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); ASSERT_TRUE(fake_upstream_connection->write("", true)); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); tcp_client->waitForDisconnect(); } @@ -369,7 +369,7 @@ TEST_P(TcpProxyIntegrationTest, ShutdownWithOpenConnections) { test_server_.reset(); ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); tcp_client->waitForHalfClose(); tcp_client->close(); @@ -397,7 +397,7 @@ TEST_P(TcpProxyIntegrationTest, TestIdletimeoutWithNoData) { initialize(); IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); - tcp_client->waitForDisconnect(true); + tcp_client->waitForDisconnect(); } TEST_P(TcpProxyIntegrationTest, TestIdletimeoutWithLargeOutstandingData) { @@ -427,8 +427,8 @@ TEST_P(TcpProxyIntegrationTest, TestIdletimeoutWithLargeOutstandingData) { ASSERT_TRUE(tcp_client->write(data)); ASSERT_TRUE(fake_upstream_connection->write(data)); - tcp_client->waitForDisconnect(true); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + tcp_client->waitForDisconnect(); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); } TEST_P(TcpProxyIntegrationTest, TestNoCloseOnHealthFailure) { @@ -475,7 +475,7 @@ TEST_P(TcpProxyIntegrationTest, TestNoCloseOnHealthFailure) { ASSERT_TRUE(fake_upstream_health_connection->waitForData(8)); ASSERT_TRUE(fake_upstream_health_connection->close()); - ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect()); // By waiting we know the previous health check attempt completed (with a failure since we closed // the connection on it) @@ -492,10 +492,10 @@ TEST_P(TcpProxyIntegrationTest, TestNoCloseOnHealthFailure) { test_server_.reset(); ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); ASSERT_TRUE(fake_upstream_connection->close()); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForHalfClose()); ASSERT_TRUE(fake_upstream_health_connection_reconnect->close()); - ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_health_connection_reconnect->waitForDisconnect()); tcp_client->waitForHalfClose(); tcp_client->close(); } @@ -545,14 +545,14 @@ TEST_P(TcpProxyIntegrationTest, TestCloseOnHealthFailure) { ASSERT_TRUE(fake_upstream_health_connection->waitForData(8)); fake_upstreams_[0]->set_allow_unexpected_disconnects(true); ASSERT_TRUE(fake_upstream_health_connection->close()); - ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_health_connection->waitForDisconnect()); ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); tcp_client->waitForHalfClose(); ASSERT_TRUE(fake_upstream_connection->close()); tcp_client->close(); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); } class TcpProxyMetadataMatchIntegrationTest : public TcpProxyIntegrationTest { @@ -636,7 +636,7 @@ void TcpProxyMetadataMatchIntegrationTest::expectEndpointToMatchRoute() { ASSERT_TRUE(fake_upstream_connection->waitForData(10)); ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); ASSERT_TRUE(fake_upstream_connection->write("", true)); - ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); tcp_client->waitForDisconnect(); test_server_->waitForCounterGe("cluster.cluster_0.lb_subsets_selected", 1); @@ -647,9 +647,9 @@ void TcpProxyMetadataMatchIntegrationTest::expectEndpointNotToMatchRoute() { IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); ASSERT_TRUE(tcp_client->write("hello", false, false)); - // TODO(yskopets): 'tcp_client->waitForDisconnect(true);' gets stuck indefinitely on Linux builds, + // TODO(yskopets): 'tcp_client->waitForDisconnect();' gets stuck indefinitely on Linux builds, // e.g. on 'envoy-linux (bazel compile_time_options)' and 'envoy-linux (bazel release)' - // tcp_client->waitForDisconnect(true); + // tcp_client->waitForDisconnect(); test_server_->waitForCounterGe("cluster.cluster_0.upstream_cx_none_healthy", 1); test_server_->waitForCounterEq("cluster.cluster_0.lb_subsets_selected", 0); diff --git a/test/integration/tcp_tunneling_integration_test.cc b/test/integration/tcp_tunneling_integration_test.cc index bc270853ae65..333e98650f04 100644 --- a/test/integration/tcp_tunneling_integration_test.cc +++ b/test/integration/tcp_tunneling_integration_test.cc @@ -396,7 +396,7 @@ TEST_P(TcpTunnelingIntegrationTest, ResetStreamTest) { // Reset the stream. upstream_request_->encodeResetStream(); - tcp_client->waitForDisconnect(true); + tcp_client->waitForDisconnect(); } TEST_P(TcpTunnelingIntegrationTest, TestIdletimeoutWithLargeOutstandingData) { @@ -429,7 +429,7 @@ TEST_P(TcpTunnelingIntegrationTest, TestIdletimeoutWithLargeOutstandingData) { ASSERT_TRUE(tcp_client->write(data)); upstream_request_->encodeData(data, false); - tcp_client->waitForDisconnect(true); + tcp_client->waitForDisconnect(); ASSERT_TRUE(upstream_request_->waitForReset()); } diff --git a/test/integration/utility.cc b/test/integration/utility.cc index c969a5b8a2ef..3468efea0d26 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -139,8 +139,10 @@ RawConnectionDriver::RawConnectionDriver(uint32_t port, Buffer::Instance& initia RawConnectionDriver::~RawConnectionDriver() = default; void RawConnectionDriver::waitForConnection() { + // TODO(mattklein123): Add a timeout and switch to events and waitFor(). while (!callbacks_->connected() && !callbacks_->closed()) { - Event::GlobalTimeSystem().timeSystem().advanceTimeWait(std::chrono::milliseconds(10)); + Event::GlobalTimeSystem().timeSystem().realSleepDoNotUseWithoutScrutiny( + std::chrono::milliseconds(10)); dispatcher_.run(Event::Dispatcher::RunType::NonBlock); } } diff --git a/test/integration/vhds_integration_test.cc b/test/integration/vhds_integration_test.cc index 879fbcf8b90b..efa74c603977 100644 --- a/test/integration/vhds_integration_test.cc +++ b/test/integration/vhds_integration_test.cc @@ -209,7 +209,7 @@ TEST_P(VhdsInitializationTest, InitializeVhdsAfterRdsHasBeenInitialized) { {TestUtility::parseYaml(RdsConfigWithVhosts)}, "2"); - auto result = xds_connection_->waitForNewStream(*dispatcher_, vhds_stream_, true); + auto result = xds_connection_->waitForNewStream(*dispatcher_, vhds_stream_); RELEASE_ASSERT(result, result.message()); vhds_stream_->startGrpcStream(); @@ -301,7 +301,7 @@ class VhdsIntegrationTest : public HttpIntegrationTest, sendSotwDiscoveryResponse( Config::TypeUrl::get().RouteConfiguration, {rdsConfig()}, "1"); - result = xds_connection_->waitForNewStream(*dispatcher_, vhds_stream_, true); + result = xds_connection_->waitForNewStream(*dispatcher_, vhds_stream_); RELEASE_ASSERT(result, result.message()); vhds_stream_->startGrpcStream(); diff --git a/test/mocks/common.h b/test/mocks/common.h index cf5a915825b2..ee29efa4c333 100644 --- a/test/mocks/common.h +++ b/test/mocks/common.h @@ -56,14 +56,11 @@ class MockTimeSystem : public Event::TestTimeSystem { Event::CallbackScheduler& cb_scheduler) override { return real_time_.createScheduler(base_scheduler, cb_scheduler); } - void advanceTimeWait(const Duration& duration) override { real_time_.advanceTimeWait(duration); } - void advanceTimeAsync(const Duration& duration) override { - real_time_.advanceTimeAsync(duration); + void advanceTimeWaitImpl(const Duration& duration) override { + real_time_.advanceTimeWaitImpl(duration); } - Thread::CondVar::WaitStatus waitFor(Thread::MutexBasicLockable& mutex, Thread::CondVar& condvar, - const Duration& duration) noexcept - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) override { - return real_time_.waitFor(mutex, condvar, duration); // NO_CHECK_FORMAT(real_time) + void advanceTimeAsyncImpl(const Duration& duration) override { + real_time_.advanceTimeAsyncImpl(duration); } MOCK_METHOD(SystemTime, systemTime, ()); MOCK_METHOD(MonotonicTime, monotonicTime, ()); diff --git a/test/test_common/BUILD b/test/test_common/BUILD index 7b0a5c972382..97be022e4d02 100644 --- a/test/test_common/BUILD +++ b/test/test_common/BUILD @@ -244,7 +244,6 @@ envoy_cc_test_library( hdrs = ["test_time.h"], deps = [ ":global_lib", - ":only_one_thread_lib", ":test_time_system_interface", "//source/common/event:real_time_system_lib", ], @@ -256,6 +255,7 @@ envoy_cc_test_library( hdrs = ["test_time_system.h"], deps = [ ":global_lib", + ":only_one_thread_lib", "//include/envoy/event:timer_interface", "//source/common/common:thread_lib", ], @@ -266,7 +266,6 @@ envoy_cc_test_library( srcs = ["simulated_time_system.cc"], hdrs = ["simulated_time_system.h"], deps = [ - ":only_one_thread_lib", ":test_time_system_interface", ":utility_lib", "//source/common/event:event_impl_base_lib", diff --git a/test/test_common/simulated_time_system.cc b/test/test_common/simulated_time_system.cc index d361beddf471..4b898ce8ef96 100644 --- a/test/test_common/simulated_time_system.cc +++ b/test/test_common/simulated_time_system.cc @@ -216,7 +216,7 @@ MonotonicTime SimulatedTimeSystemHelper::monotonicTime() { return monotonic_time_; } -void SimulatedTimeSystemHelper::advanceTimeAsync(const Duration& duration) { +void SimulatedTimeSystemHelper::advanceTimeAsyncImpl(const Duration& duration) { only_one_thread_.checkOneThread(); absl::MutexLock lock(&mutex_); MonotonicTime monotonic_time = @@ -224,7 +224,7 @@ void SimulatedTimeSystemHelper::advanceTimeAsync(const Duration& duration) { setMonotonicTimeLockHeld(monotonic_time); } -void SimulatedTimeSystemHelper::advanceTimeWait(const Duration& duration) { +void SimulatedTimeSystemHelper::advanceTimeWaitImpl(const Duration& duration) { only_one_thread_.checkOneThread(); absl::MutexLock lock(&mutex_); MonotonicTime monotonic_time = @@ -240,55 +240,6 @@ void SimulatedTimeSystemHelper::waitForNoPendingLockHeld() const &pending_alarms_)); } -Thread::CondVar::WaitStatus SimulatedTimeSystemHelper::waitFor(Thread::MutexBasicLockable& mutex, - Thread::CondVar& condvar, - const Duration& duration) noexcept - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) { - only_one_thread_.checkOneThread(); - - // TODO(#10568): This real-time polling delay should not be necessary. Without - // it, test/extensions/filters/http/cache:cache_filter_integration_test fails - // about 40% of the time. - const Duration real_time_poll_delay( - std::min(std::chrono::duration_cast(std::chrono::milliseconds(50)), duration)); - const MonotonicTime end_time = monotonicTime() + duration; - - bool timeout_not_reached = true; - while (timeout_not_reached) { - // First check to see if the condition is already satisfied without advancing sim time. - if (condvar.waitFor(mutex, real_time_poll_delay) == Thread::CondVar::WaitStatus::NoTimeout) { - return Thread::CondVar::WaitStatus::NoTimeout; - } - - // This function runs with the caller-provided mutex held. We need to - // hold this->mutex_ while accessing the timer-queue and blocking on - // callbacks completing. To avoid potential deadlock we must drop - // the caller's mutex before taking ours. We also must care to avoid - // break/continue/return/throw during this non-RAII lock operation. - mutex.unlock(); - { - absl::MutexLock lock(&mutex_); - if (monotonic_time_ < end_time) { - MonotonicTime next_wakeup = end_time; - if (!alarms_.empty()) { - // If there's another alarm pending, sleep forward to it. - const AlarmRegistration& alarm_registration = *alarms_.begin(); - next_wakeup = std::min(alarm_registration.time_, next_wakeup); - } - setMonotonicTimeLockHeld(next_wakeup); - waitForNoPendingLockHeld(); - } else { - // If we reached our end_time, break the loop and return timeout. We - // don't break immediately as we have to drop mutex_ and re-take mutex, - // and it's cleaner to have a linear flow to the end of the loop. - timeout_not_reached = false; - } - } - mutex.lock(); - } - return Thread::CondVar::WaitStatus::Timeout; -} - void SimulatedTimeSystemHelper::alarmActivateLockHeld(Alarm& alarm) ABSL_NO_THREAD_SAFETY_ANALYSIS { // We disable thread-safety analysis as the compiler can't detect that // alarm_.timeSystem() == this, so we must be holding the right mutex. diff --git a/test/test_common/simulated_time_system.h b/test/test_common/simulated_time_system.h index e8a369e4f9cc..70b0e6a6b41f 100644 --- a/test/test_common/simulated_time_system.h +++ b/test/test_common/simulated_time_system.h @@ -6,7 +6,6 @@ #include "common/common/thread.h" #include "common/common/utility.h" -#include "test/test_common/only_one_thread.h" #include "test/test_common/test_time_system.h" #include "test/test_common/utility.h" @@ -29,11 +28,8 @@ class SimulatedTimeSystemHelper : public TestTimeSystem { SchedulerPtr createScheduler(Scheduler& base_scheduler, CallbackScheduler& cb_scheduler) override; // TestTimeSystem - void advanceTimeWait(const Duration& duration) override; - void advanceTimeAsync(const Duration& duration) override; - Thread::CondVar::WaitStatus waitFor(Thread::MutexBasicLockable& mutex, Thread::CondVar& condvar, - const Duration& duration) noexcept - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) override; + void advanceTimeWaitImpl(const Duration& duration) override; + void advanceTimeAsyncImpl(const Duration& duration) override; // TimeSource SystemTime systemTime() override; @@ -137,7 +133,6 @@ class SimulatedTimeSystemHelper : public TestTimeSystem { alarm_registrations_map_ ABSL_GUARDED_BY(mutex_); mutable absl::Mutex mutex_; uint32_t pending_alarms_ ABSL_GUARDED_BY(mutex_); - Thread::OnlyOneThread only_one_thread_; }; // Represents a simulated time system, where time is advanced by calling diff --git a/test/test_common/simulated_time_system_test.cc b/test/test_common/simulated_time_system_test.cc index 13a435148aff..4dd55c610fa4 100644 --- a/test/test_common/simulated_time_system_test.cc +++ b/test/test_common/simulated_time_system_test.cc @@ -289,60 +289,57 @@ TEST_P(SimulatedTimeSystemTest, WaitFor) { EXPECT_EQ(start_system_time_, time_system_.systemTime()); // Run an event loop in the background to activate timers. - std::atomic done(false); - auto thread = Thread::threadFactoryForTest().createThread([this, &done]() { - while (!done) { + absl::Mutex mutex; + bool done(false); + auto thread = Thread::threadFactoryForTest().createThread([this, &mutex, &done]() { + for (;;) { + { + absl::MutexLock lock(&mutex); + if (done) { + return; + } + } + base_scheduler_.run(Dispatcher::RunType::Block); } }); - Thread::MutexBasicLockable mutex; - Thread::CondVar condvar; + TimerPtr timer = scheduler_->createTimer( - [&condvar, &mutex, &done]() { - Thread::LockGuard lock(mutex); + [&mutex, &done]() { + absl::MutexLock lock(&mutex); done = true; - condvar.notifyOne(); }, dispatcher_); timer->enableTimer(std::chrono::seconds(60)); - // Wait 50 simulated seconds of simulated time, which won't be enough to - // activate the alarm. We'll get a fast automatic timeout in waitFor because - // there are no pending timers. + // Wait 1ms of real time. waitFor() does not advance simulated time, so this is just going to + // verify that we return quickly and nothing has fired. { - Thread::LockGuard lock(mutex); - EXPECT_EQ(Thread::CondVar::WaitStatus::Timeout, - time_system_.waitFor(mutex, condvar, std::chrono::seconds(50))); + absl::MutexLock lock(&mutex); + EXPECT_FALSE(time_system_.waitFor(mutex, absl::Condition(&done), std::chrono::milliseconds(1))); } EXPECT_FALSE(done); - EXPECT_EQ(MonotonicTime(std::chrono::seconds(50)), time_system_.monotonicTime()); + EXPECT_EQ(MonotonicTime(std::chrono::seconds(0)), time_system_.monotonicTime()); - // Waiting another 20 simulated seconds will activate the alarm after 10, - // and the event-loop thread will call the corresponding callback quickly. + // Fire the timeout by advancing time and then verify that waitFor() returns without any timeout. + time_system_.advanceTimeWait(std::chrono::seconds(60)); { - Thread::LockGuard lock(mutex); - // We don't check for the return value of waitFor() as it can spuriously - // return timeout even if the condition is satisfied before entering into - // the waitFor(). - // - // TODO(jmarantz): just drop the return value in the API. - time_system_.waitFor(mutex, condvar, std::chrono::seconds(10)); + absl::MutexLock lock(&mutex); + EXPECT_TRUE(time_system_.waitFor(mutex, absl::Condition(&done), std::chrono::seconds(0))); } EXPECT_TRUE(done); EXPECT_EQ(MonotonicTime(std::chrono::seconds(60)), time_system_.monotonicTime()); + thread->join(); // Waiting a third time, with no pending timeouts, will just sleep out for // the max duration and return a timeout. done = false; { - Thread::LockGuard lock(mutex); - EXPECT_EQ(Thread::CondVar::WaitStatus::Timeout, - time_system_.waitFor(mutex, condvar, std::chrono::seconds(20))); + absl::MutexLock lock(&mutex); + EXPECT_FALSE(time_system_.waitFor(mutex, absl::Condition(&done), std::chrono::seconds(0))); } EXPECT_FALSE(done); - EXPECT_EQ(MonotonicTime(std::chrono::seconds(80)), time_system_.monotonicTime()); - - thread->join(); + EXPECT_EQ(MonotonicTime(std::chrono::seconds(60)), time_system_.monotonicTime()); } TEST_P(SimulatedTimeSystemTest, Monotonic) { @@ -442,26 +439,39 @@ TEST_P(SimulatedTimeSystemTest, DuplicateTimer) { EXPECT_EQ("2", output_); // Now set an alarm which requires 10ms of progress and make sure waitFor works. - std::atomic done(false); - auto thread = Thread::threadFactoryForTest().createThread([this, &done]() { - while (!done) { + absl::Mutex mutex; + bool done(false); + auto thread = Thread::threadFactoryForTest().createThread([this, &mutex, &done]() { + for (;;) { + { + absl::MutexLock lock(&mutex); + if (done) { + return; + } + } + base_scheduler_.run(Dispatcher::RunType::Block); } }); - Thread::MutexBasicLockable mutex; - Thread::CondVar condvar; + TimerPtr timer = scheduler_->createTimer( - [&condvar, &mutex, &done]() { - Thread::LockGuard lock(mutex); + [&mutex, &done]() { + absl::MutexLock lock(&mutex); done = true; - condvar.notifyOne(); }, dispatcher_); timer->enableTimer(std::chrono::seconds(10)); { - Thread::LockGuard lock(mutex); - time_system_.waitFor(mutex, condvar, std::chrono::seconds(10)); + absl::MutexLock lock(&mutex); + EXPECT_FALSE(time_system_.waitFor(mutex, absl::Condition(&done), std::chrono::seconds(0))); + } + EXPECT_FALSE(done); + + time_system_.advanceTimeWait(std::chrono::seconds(10)); + { + absl::MutexLock lock(&mutex); + EXPECT_TRUE(time_system_.waitFor(mutex, absl::Condition(&done), std::chrono::seconds(0))); } EXPECT_TRUE(done); diff --git a/test/test_common/test_time.cc b/test/test_common/test_time.cc index d5a50fd1de91..98deac1a959d 100644 --- a/test/test_common/test_time.cc +++ b/test/test_common/test_time.cc @@ -18,18 +18,13 @@ TestTimeSystem& GlobalTimeSystem::timeSystem() { return singleton_->timeSystem(make_real_time_system); } -void TestRealTimeSystem::advanceTimeWait(const Duration& duration) { +void TestRealTimeSystem::advanceTimeWaitImpl(const Duration& duration) { only_one_thread_.checkOneThread(); std::this_thread::sleep_for(duration); } -void TestRealTimeSystem::advanceTimeAsync(const Duration& duration) { advanceTimeWait(duration); } - -Thread::CondVar::WaitStatus TestRealTimeSystem::waitFor(Thread::MutexBasicLockable& lock, - Thread::CondVar& condvar, - const Duration& duration) noexcept { - only_one_thread_.checkOneThread(); - return condvar.waitFor(lock, duration); +void TestRealTimeSystem::advanceTimeAsyncImpl(const Duration& duration) { + advanceTimeWait(duration); } SystemTime TestRealTimeSystem::systemTime() { return real_time_system_.systemTime(); } diff --git a/test/test_common/test_time.h b/test/test_common/test_time.h index 31880b73b5e2..038a44e734c5 100644 --- a/test/test_common/test_time.h +++ b/test/test_common/test_time.h @@ -3,7 +3,6 @@ #include "common/event/real_time_system.h" #include "test/test_common/global.h" -#include "test/test_common/only_one_thread.h" #include "test/test_common/test_time_system.h" namespace Envoy { @@ -12,11 +11,8 @@ namespace Event { class TestRealTimeSystem : public TestTimeSystem { public: // TestTimeSystem - void advanceTimeAsync(const Duration& duration) override; - void advanceTimeWait(const Duration& duration) override; - Thread::CondVar::WaitStatus waitFor(Thread::MutexBasicLockable& mutex, Thread::CondVar& condvar, - const Duration& duration) noexcept - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) override; + void advanceTimeAsyncImpl(const Duration& duration) override; + void advanceTimeWaitImpl(const Duration& duration) override; // Event::TimeSystem Event::SchedulerPtr createScheduler(Scheduler& base_scheduler, @@ -30,7 +26,6 @@ class TestRealTimeSystem : public TestTimeSystem { private: Event::RealTimeSystem real_time_system_; - Thread::OnlyOneThread only_one_thread_; }; class GlobalTimeSystem : public DelegatingTestTimeSystemBase { diff --git a/test/test_common/test_time_system.h b/test/test_common/test_time_system.h index bc5d38972879..4cbd3d5c4f39 100644 --- a/test/test_common/test_time_system.h +++ b/test/test_common/test_time_system.h @@ -1,11 +1,13 @@ #pragma once +#include "envoy/common/time.h" #include "envoy/event/timer.h" #include "common/common/assert.h" #include "common/common/thread.h" #include "test/test_common/global.h" +#include "test/test_common/only_one_thread.h" namespace Envoy { namespace Event { @@ -15,6 +17,34 @@ class TestTimeSystem : public Event::TimeSystem { public: ~TestTimeSystem() override = default; + /** + * This class will use the real monotonic time regardless of the time system in use (real + * or simulated). This should only be used when time is needed for real timeouts that govern + * networking, etc. It should never be used for time that only advances explicitly for alarms. + */ + class RealTimeBound { + public: + template + RealTimeBound(const D& duration) + : end_time_(std::chrono::steady_clock::now() + duration) // NO_CHECK_FORMAT(real_time) + {} + + std::chrono::milliseconds timeLeft() { + const auto current_time = std::chrono::steady_clock::now(); // NO_CHECK_FORMAT(real_time) + if (current_time > end_time_) { + return std::chrono::milliseconds(0); + } + return std::chrono::duration_cast(end_time_ - current_time); + } + + bool withinBound() { + return std::chrono::steady_clock::now() < end_time_; // NO_CHECK_FORMAT(real_time) + } + + private: + const MonotonicTime end_time_; + }; + /** * Advances time forward by the specified duration, running any timers * scheduled to fire, and blocking until the timer callbacks are complete. @@ -26,9 +56,9 @@ class TestTimeSystem : public Event::TimeSystem { * * @param duration The amount of time to sleep. */ - virtual void advanceTimeWait(const Duration& duration) PURE; - template void advanceTimeWait(const D& duration) { - advanceTimeWait(std::chrono::duration_cast(duration)); + virtual void advanceTimeWaitImpl(const Duration& duration) PURE; + template void advanceTimeWait(const D& duration = false) { + advanceTimeWaitImpl(std::chrono::duration_cast(duration)); } /** @@ -42,31 +72,46 @@ class TestTimeSystem : public Event::TimeSystem { * * @param duration The amount of time to sleep. */ - virtual void advanceTimeAsync(const Duration& duration) PURE; - template void advanceTimeAsync(const D& duration) { - advanceTimeAsync(std::chrono::duration_cast(duration)); + virtual void advanceTimeAsyncImpl(const Duration& duration) PURE; + template void advanceTimeAsync(const D& duration = false) { + advanceTimeAsyncImpl(std::chrono::duration_cast(duration)); } /** - * Waits for the specified duration to expire, or for a condvar to - * be notified, whichever comes first. + * Waits for the specified duration to expire, or for the condition to be satisfied, whichever + * comes first. + * + * NOTE: This function takes a duration parameter which is the timeout of the wait. This is *real* + * time in all time systems. This is to avoid test hangs and provide a useful error message. + * When using simulated time this does not advance monotonic time. Thus, to simulated time + * tests all network behavior will appear instantaneous. If time needs to advance to fire + * alarms advanceTimeWait() or advanceTimeAsync() should be used. * * @param mutex A mutex which must be held before calling this function. - * @param condvar The condition to wait on. + * @param condition The condition to wait on. * @param duration The maximum amount of time to wait. * @return Thread::CondVar::WaitStatus whether the condition timed out or not. */ - virtual Thread::CondVar::WaitStatus waitFor(Thread::MutexBasicLockable& mutex, - Thread::CondVar& condvar, - const Duration& duration) noexcept - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) PURE; - template - Thread::CondVar::WaitStatus waitFor(Thread::MutexBasicLockable& mutex, Thread::CondVar& condvar, - const D& duration) noexcept + bool waitFor(absl::Mutex& mutex, const absl::Condition& condition, const D& duration) noexcept ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) { - return waitFor(mutex, condvar, std::chrono::duration_cast(duration)); + only_one_thread_.checkOneThread(); + return mutex.AwaitWithTimeout(condition, + absl::FromChrono(std::chrono::duration_cast(duration))); } + + /** + * This function will perform a real sleep in all time systems (real or simulated). This function + * should NOT be used without a good reason. It is either for supporting old code that needs to + * be converted to an event based approach, simulated time, or some other solution. Be ready + * to explain why you are using this in code review. + */ + template void realSleepDoNotUseWithoutScrutiny(const D& duration) { + std::this_thread::sleep_for(duration); // NO_CHECK_FORMAT(real_time) + } + +protected: + Thread::OnlyOneThread only_one_thread_; }; // There should only be one instance of any time-system resident in a test @@ -102,19 +147,12 @@ class SingletonTimeSystemHelper { // subclass. template class DelegatingTestTimeSystemBase : public TestTimeSystem { public: - void advanceTimeAsync(const Duration& duration) override { - timeSystem().advanceTimeAsync(duration); + void advanceTimeAsyncImpl(const Duration& duration) override { + timeSystem().advanceTimeAsyncImpl(duration); } - void advanceTimeWait(const Duration& duration) override { - timeSystem().advanceTimeWait(duration); + void advanceTimeWaitImpl(const Duration& duration) override { + timeSystem().advanceTimeWaitImpl(duration); } - - Thread::CondVar::WaitStatus waitFor(Thread::MutexBasicLockable& mutex, Thread::CondVar& condvar, - const Duration& duration) noexcept - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex) override { - return timeSystem().waitFor(mutex, condvar, duration); - } - SchedulerPtr createScheduler(Scheduler& base_scheduler, CallbackScheduler& cb_scheduler) override { return timeSystem().createScheduler(base_scheduler, cb_scheduler); diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 13feb48e5df5..0909ac0bc44a 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -160,13 +160,17 @@ Stats::TextReadoutSharedPtr TestUtility::findTextReadout(Stats::Store& store, AssertionResult TestUtility::waitForCounterEq(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system, - std::chrono::milliseconds timeout) { - auto end_time = time_system.monotonicTime() + timeout; + std::chrono::milliseconds timeout, + Event::Dispatcher* dispatcher) { + Event::TestTimeSystem::RealTimeBound bound(timeout); while (findCounter(store, name) == nullptr || findCounter(store, name)->value() != value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (timeout != std::chrono::milliseconds::zero() && time_system.monotonicTime() >= end_time) { + if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } + if (dispatcher != nullptr) { + dispatcher->run(Event::Dispatcher::RunType::NonBlock); + } } return AssertionSuccess(); } @@ -174,10 +178,10 @@ AssertionResult TestUtility::waitForCounterEq(Stats::Store& store, const std::st AssertionResult TestUtility::waitForCounterGe(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { - auto end_time = time_system.monotonicTime() + timeout; + Event::TestTimeSystem::RealTimeBound bound(timeout); while (findCounter(store, name) == nullptr || findCounter(store, name)->value() < value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (timeout != std::chrono::milliseconds::zero() && time_system.monotonicTime() >= end_time) { + if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } } @@ -187,10 +191,10 @@ AssertionResult TestUtility::waitForCounterGe(Stats::Store& store, const std::st AssertionResult TestUtility::waitForGaugeGe(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { - auto end_time = time_system.monotonicTime() + timeout; + Event::TestTimeSystem::RealTimeBound bound(timeout); while (findGauge(store, name) == nullptr || findGauge(store, name)->value() < value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (timeout != std::chrono::milliseconds::zero() && time_system.monotonicTime() >= end_time) { + if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } } @@ -200,10 +204,10 @@ AssertionResult TestUtility::waitForGaugeGe(Stats::Store& store, const std::stri AssertionResult TestUtility::waitForGaugeEq(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system, std::chrono::milliseconds timeout) { - auto end_time = time_system.monotonicTime() + timeout; + Event::TestTimeSystem::RealTimeBound bound(timeout); while (findGauge(store, name) == nullptr || findGauge(store, name)->value() != value) { time_system.advanceTimeWait(std::chrono::milliseconds(10)); - if (timeout != std::chrono::milliseconds::zero() && time_system.monotonicTime() >= end_time) { + if (timeout != std::chrono::milliseconds::zero() && !bound.withinBound()) { return AssertionFailure() << fmt::format("timed out waiting for {} to be {}", name, value); } } @@ -214,7 +218,6 @@ std::list TestUtility::makeDnsResponse(const std::list& addresses, std::chrono::seconds ttl) { std::list ret; for (const auto& address : addresses) { - ret.emplace_back(Network::DnsResponse(Network::Utility::parseInternetAddress(address), ttl)); } return ret; @@ -367,29 +370,27 @@ bool TestUtility::gaugesZeroed( } void ConditionalInitializer::setReady() { - Thread::LockGuard lock(mutex_); + absl::MutexLock lock(&mutex_); EXPECT_FALSE(ready_); ready_ = true; - cv_.notifyAll(); } void ConditionalInitializer::waitReady() { - Thread::LockGuard lock(mutex_); + absl::MutexLock lock(&mutex_); if (ready_) { ready_ = false; return; } - cv_.wait(mutex_); + mutex_.Await(absl::Condition(&ready_)); EXPECT_TRUE(ready_); ready_ = false; } void ConditionalInitializer::wait() { - Thread::LockGuard lock(mutex_); - while (!ready_) { - cv_.wait(mutex_); - } + absl::MutexLock lock(&mutex_); + mutex_.Await(absl::Condition(&ready_)); + EXPECT_TRUE(ready_); } constexpr std::chrono::milliseconds TestUtility::DefaultTimeout; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index df96fff5d7f6..8c1d983c3ada 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -201,13 +201,15 @@ class TestUtility { * @param value supplies the value of the counter. * @param time_system the time system to use for waiting. * @param timeout the maximum time to wait before timing out, or 0 for no timeout. + * @param dispatcher the dispatcher to run non-blocking periodically during the wait. * @return AssertionSuccess() if the counter was == to the value within the timeout, else * AssertionFailure(). */ static AssertionResult waitForCounterEq(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system, - std::chrono::milliseconds timeout = std::chrono::milliseconds::zero()); + std::chrono::milliseconds timeout = std::chrono::milliseconds::zero(), + Event::Dispatcher* dispatcher = nullptr); /** * Wait for a counter to >= a given value. @@ -792,9 +794,8 @@ class ConditionalInitializer { void wait(); private: - Thread::CondVar cv_; - Thread::MutexBasicLockable mutex_; - bool ready_{false}; + absl::Mutex mutex_; + bool ready_ ABSL_GUARDED_BY(mutex_){false}; }; namespace Http { From 62f7d931ace4ac2b4a872d2a2c37969eff6a5308 Mon Sep 17 00:00:00 2001 From: Arthur Yan <55563955+arthuryan-k@users.noreply.github.com> Date: Fri, 14 Aug 2020 11:43:47 -0400 Subject: [PATCH 62/67] fuzz: added fuzz test for listener filter tls_inspector (#12617) Created tls_inspector_corpus and populated with testcases (valid and invalid client hellos) Risk Level: Low Testing: increased function coverage of tls_inspector.cc to 100.0% and line coverage to 87.3% after running fuzzer (covers all parse states except errors related to socket read failure). Docs Changes: N/A Release Notes: N/A Signed-off-by: Arthur Yan --- .../common/fuzz/listener_filter_fuzzer.cc | 9 ++-- .../common/fuzz/listener_filter_fuzzer.h | 6 +-- .../common/fuzz/listener_filter_fuzzer.proto | 2 +- .../filters/listener/tls_inspector/BUILD | 21 ++++++++++ .../tls_inspector_corpus/multiple_reads | 4 ++ .../tls_inspector_corpus/no_extensions | 3 ++ .../tls_inspector_corpus/not_ssl | 3 ++ .../tls_inspector_corpus/too_large | 5 +++ .../tls_inspector_corpus/valid_alpn | 3 ++ .../tls_inspector_corpus/valid_sni | 3 ++ .../tls_inspector/tls_inspector_fuzz_test.cc | 41 +++++++++++++++++++ .../tls_inspector_fuzz_test.proto | 12 ++++++ 12 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/multiple_reads create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/no_extensions create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/not_ssl create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/too_large create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/valid_alpn create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/valid_sni create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_fuzz_test.cc create mode 100644 test/extensions/filters/listener/tls_inspector/tls_inspector_fuzz_test.proto diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc index 0f5aa60b8d44..ef9481d2190c 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.cc @@ -60,6 +60,7 @@ void ListenerFilterFuzzer::fuzz( while (!got_continue) { if (header.done()) { // End of stream reached but not done file_event_callback_(Event::FileReadyType::Closed); + return; } else { file_event_callback_(Event::FileReadyType::Read); } @@ -74,11 +75,11 @@ FuzzedHeader::FuzzedHeader(const test::extensions::filters::listener::FilterFuzz len += input.data(i).size(); } - header_.reserve(len); + data_.reserve(len); for (int i = 0; i < nreads_; i++) { - header_ += input.data(i); - indices_.push_back(header_.size()); + data_.insert(data_.end(), input.data(i).begin(), input.data(i).end()); + indices_.push_back(data_.size()); } } @@ -86,7 +87,7 @@ Api::SysCallSizeResult FuzzedHeader::next(void* buffer, size_t length) { if (done()) { // End of stream reached nread_ = nreads_ - 1; // Decrement to avoid out-of-range for last recv() call } - memcpy(buffer, header_.data(), std::min(indices_[nread_], length)); + memcpy(buffer, data_.data(), std::min(indices_[nread_], length)); return Api::SysCallSizeResult{static_cast(indices_[nread_++]), 0}; } diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h index 66b6f8707bfd..d44f9d86f91f 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.h @@ -44,9 +44,9 @@ class FuzzedHeader { bool empty(); private: - const int nreads_; // Number of reads - int nread_; // Counter of current read - std::string header_; // Construct header from single or multiple reads + const int nreads_; // Number of reads + int nread_; // Counter of current read + std::vector data_; std::vector indices_; // Ending indices for each read }; diff --git a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto index 5741ed9edfa3..0f7bf2e5369e 100644 --- a/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto +++ b/test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto @@ -9,5 +9,5 @@ message Socket { message FilterFuzzTestCase { Socket sock = 1; - repeated string data = 2; + repeated bytes data = 2; } \ No newline at end of file diff --git a/test/extensions/filters/listener/tls_inspector/BUILD b/test/extensions/filters/listener/tls_inspector/BUILD index 0f654911f672..2ed4fdeb984c 100644 --- a/test/extensions/filters/listener/tls_inspector/BUILD +++ b/test/extensions/filters/listener/tls_inspector/BUILD @@ -1,8 +1,10 @@ load( "//bazel:envoy_build_system.bzl", + "envoy_cc_fuzz_test", "envoy_cc_library", "envoy_cc_test", "envoy_package", + "envoy_proto_library", ) load( "//test/extensions:extensions_build_system.bzl", @@ -29,6 +31,25 @@ envoy_cc_test( ], ) +envoy_proto_library( + name = "tls_inspector_fuzz_test_proto", + srcs = ["tls_inspector_fuzz_test.proto"], + deps = [ + "//test/extensions/filters/listener/common/fuzz:listener_filter_fuzzer_proto", + ], +) + +envoy_cc_fuzz_test( + name = "tls_inspector_fuzz_test", + srcs = ["tls_inspector_fuzz_test.cc"], + corpus = "tls_inspector_corpus", + deps = [ + ":tls_inspector_fuzz_test_proto_cc_proto", + "//source/extensions/filters/listener/tls_inspector:tls_inspector_lib", + "//test/extensions/filters/listener/common/fuzz:listener_filter_fuzzer_lib", + ], +) + envoy_extension_cc_benchmark_binary( name = "tls_inspector_benchmark", srcs = ["tls_inspector_benchmark.cc"], diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/multiple_reads b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/multiple_reads new file mode 100644 index 000000000000..f6b2fa1175f5 --- /dev/null +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/multiple_reads @@ -0,0 +1,4 @@ +fuzzed { + data: "\x16\x03\x01\x00\xae\x01\x00\x00\xaa\x03\x03V\xca\x92\x12\xa1\x07II\xc2e\'\x10\x1ajm;Nz\x87\xd6\x00\x17X&\x81\xc4\x95\xb9_5\xc5w A\"\xc3\x1a\xf1\xc6\xaa=\x9a\x83\x9f\x11w\x1eW\xdf\x960\x04\xd0|\xc5\xb4\x88\xa5\xc0\x9e.*\x8e\xcf\xf5\x00\x06\x13\x01" + data: "\x13\x02\x13\x03\x01\x00\x00[\x00\n\x00\x08\x00\x06\x00\x1d\x00\x17\x00\x18\x00\r\x00\x14\x00\x12\x04\x03\x08\x04\x04\x01\x05\x03\x08\x05\x05\x01\x08\x06\x06\x01\x02\x01\x003\x00&\x00$\x00\x1d\x00 F8{\xd6X\xda\xa4\x15\xe7g\xf2\\p\x92\xc5\xc2\xa8L\xfe\x9eU\x1dac\xde6\x9dm_\x04zy\x00-\x00\x02\x01\x01\x00+\x00\x03\x02\x03\x04" +} \ No newline at end of file diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/no_extensions b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/no_extensions new file mode 100644 index 000000000000..250bf8825e19 --- /dev/null +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/no_extensions @@ -0,0 +1,3 @@ +fuzzed { + data: "\x16\x03\x01\x00\xae\x01\x00\x00\xaa\x03\x03V\xca\x92\x12\xa1\x07II\xc2e\'\x10\x1ajm;Nz\x87\xd6\x00\x17X&\x81\xc4\x95\xb9_5\xc5w A\"\xc3\x1a\xf1\xc6\xaa=\x9a\x83\x9f\x11w\x1eW\xdf\x960\x04\xd0|\xc5\xb4\x88\xa5\xc0\x9e.*\x8e\xcf\xf5\x00\x06\x13\x01\x13\x02\x13\x03\x01\x00\x00[\x00\n\x00\x08\x00\x06\x00\x1d\x00\x17\x00\x18\x00\r\x00\x14\x00\x12\x04\x03\x08\x04\x04\x01\x05\x03\x08\x05\x05\x01\x08\x06\x06\x01\x02\x01\x003\x00&\x00$\x00\x1d\x00 F8{\xd6X\xda\xa4\x15\xe7g\xf2\\p\x92\xc5\xc2\xa8L\xfe\x9eU\x1dac\xde6\x9dm_\x04zy\x00-\x00\x02\x01\x01\x00+\x00\x03\x02\x03\x04" +} \ No newline at end of file diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/not_ssl b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/not_ssl new file mode 100644 index 000000000000..720e405d425c --- /dev/null +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/not_ssl @@ -0,0 +1,3 @@ +fuzzed { + data: "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" +} \ No newline at end of file diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/too_large b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/too_large new file mode 100644 index 000000000000..8d07cef375cc --- /dev/null +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_corpus/too_large @@ -0,0 +1,5 @@ +max_size: 5 +fuzzed { + data: "\x16\x03\x01\x00\xc2\x01\x00\x00\xbe\x03\x03<\xd0Be\x8d\xc5_\x06\x0e\x13\xad(store); + } else { + cfg = std::make_shared(store, input.max_size()); + } + + auto filter = std::make_unique(cfg); + + ListenerFilterFuzzer fuzzer; + fuzzer.fuzz(*filter, input.fuzzed()); +} + +} // namespace TlsInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_fuzz_test.proto b/test/extensions/filters/listener/tls_inspector/tls_inspector_fuzz_test.proto new file mode 100644 index 000000000000..ca37bbc9ed41 --- /dev/null +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_fuzz_test.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package test.extensions.filters.listener.tls_inspector; + +import "test/extensions/filters/listener/common/fuzz/listener_filter_fuzzer.proto"; +import "validate/validate.proto"; + +message TlsInspectorTestCase { + uint32 max_size = 1 [(validate.rules).uint32.lte = 65536]; + test.extensions.filters.listener.FilterFuzzTestCase fuzzed = 2 + [(validate.rules).message.required = true]; +} \ No newline at end of file From 5a602e8de707aae29ce1ada66849e4fa15c2f5a1 Mon Sep 17 00:00:00 2001 From: chaoqin-li1123 <55518381+chaoqin-li1123@users.noreply.github.com> Date: Fri, 14 Aug 2020 10:50:27 -0500 Subject: [PATCH 63/67] scoped_rds_integration_test migrate from api v2 to api v3. (#12633) Migrate the integration test of scoped rds from api v2 to api v3. Fix a bug in scoped_rds.cc: ScopedRdsConfigSubscription should use the resource version of srds, not the resource version of rds. Risk Level: Low Signed-off-by: chaoqinli --- source/common/router/scoped_rds.cc | 2 +- .../scoped_rds_integration_test.cc | 50 +++++++++++++------ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index c7e4d3db44ff..8d7c7566b764 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -101,7 +101,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, factory_context), Envoy::Config::SubscriptionBase( - rds_config_source.resource_api_version(), + scoped_rds.scoped_rds_config_source().resource_api_version(), factory_context.messageValidationContext().dynamicValidationVisitor(), "name"), factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 59dc3069d32d..7ad827d145d4 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -77,17 +77,34 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, scoped_routes->set_name(srds_config_name_); *scoped_routes->mutable_scope_key_builder() = scope_key_builder; + // Set resource api version for rds. + envoy::config::core::v3::ConfigSource* rds_config_source = + scoped_routes->mutable_rds_config_source(); + rds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + + // Set transport api version for rds. envoy::config::core::v3::ApiConfigSource* rds_api_config_source = - scoped_routes->mutable_rds_config_source()->mutable_api_config_source(); + rds_config_source->mutable_api_config_source(); + rds_api_config_source->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); + + // Add grpc service for rds. rds_api_config_source->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); envoy::config::core::v3::GrpcService* grpc_service = rds_api_config_source->add_grpc_services(); setGrpcService(*grpc_service, "rds_cluster", getRdsFakeUpstream().localAddress()); + // Set resource api version for scoped rds. + envoy::config::core::v3::ConfigSource* srds_config_source = + scoped_routes->mutable_scoped_rds()->mutable_scoped_rds_config_source(); + srds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + + // Set Transport api version for scoped_rds. envoy::config::core::v3::ApiConfigSource* srds_api_config_source = - scoped_routes->mutable_scoped_rds() - ->mutable_scoped_rds_config_source() - ->mutable_api_config_source(); + srds_config_source->mutable_api_config_source(); + srds_api_config_source->set_transport_api_version( + envoy::config::core::v3::ApiVersion::V3); + + // Add grpc service for scoped rds. if (isDelta()) { srds_api_config_source->set_api_type( envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); @@ -161,12 +178,14 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, } void sendRdsResponse(const std::string& route_config, const std::string& version) { - API_NO_BOOST(envoy::api::v2::DiscoveryResponse) response; + envoy::service::discovery::v3::DiscoveryResponse response; + std::string route_conguration_type_url = + "type.googleapis.com/envoy.config.route.v3.RouteConfiguration"; response.set_version_info(version); - response.set_type_url(Config::TypeUrl::get().RouteConfiguration); + response.set_type_url(route_conguration_type_url); auto route_configuration = TestUtility::parseYaml(route_config); - response.add_resources()->PackFrom(API_DOWNGRADE(route_configuration)); + response.add_resources()->PackFrom(route_configuration); ASSERT(rds_upstream_info_.stream_by_resource_name_[route_configuration.name()] != nullptr); rds_upstream_info_.stream_by_resource_name_[route_configuration.name()]->sendGrpcMessage( response); @@ -187,10 +206,11 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, const std::vector& to_delete_list, const std::string& version) { ASSERT(scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_] != nullptr); - - API_NO_BOOST(envoy::api::v2::DeltaDiscoveryResponse) response; + std::string scoped_route_configuration_type_url = + "type.googleapis.com/envoy.config.route.v3.ScopedRouteConfiguration"; + envoy::service::discovery::v3::DeltaDiscoveryResponse response; response.set_system_version_info(version); - response.set_type_url(Config::TypeUrl::get().ScopedRouteConfiguration); + response.set_type_url(scoped_route_configuration_type_url); for (const auto& scope_name : to_delete_list) { *response.add_removed_resources() = scope_name; @@ -201,7 +221,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, auto resource = response.add_resources(); resource->set_name(scoped_route_proto.name()); resource->set_version(version); - resource->mutable_resource()->PackFrom(API_DOWNGRADE(scoped_route_proto)); + resource->mutable_resource()->PackFrom(scoped_route_proto); } scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_]->sendGrpcMessage( response); @@ -211,14 +231,16 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, const std::string& version) { ASSERT(scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_] != nullptr); - API_NO_BOOST(envoy::api::v2::DiscoveryResponse) response; + std::string scoped_route_configuration_type_url = + "type.googleapis.com/envoy.config.route.v3.ScopedRouteConfiguration"; + envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); - response.set_type_url(Config::TypeUrl::get().ScopedRouteConfiguration); + response.set_type_url(scoped_route_configuration_type_url); for (const auto& resource_proto : resource_protos) { envoy::config::route::v3::ScopedRouteConfiguration scoped_route_proto; TestUtility::loadFromYaml(resource_proto, scoped_route_proto); - response.add_resources()->PackFrom(API_DOWNGRADE(scoped_route_proto)); + response.add_resources()->PackFrom(scoped_route_proto); } scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_]->sendGrpcMessage( response); From 630511e4b83da462d4518c0652c2520382a0e79b Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Fri, 14 Aug 2020 13:26:57 -0400 Subject: [PATCH 64/67] add 'explicit' restriction. (#12643) Commit Message: The intent, when providing Stats::Utility::counterFromElements, is that dynamic segments should be easy to construct, but still searchable. We should be trying to avoid dynamic segments whenever possible, so having them implicitly created from string data is not idea. Additional Description: Risk Level: none for the repo, but possibly will require trivial edits outside the repo Testing: //test/... Docs Changes: n/a Release Notes: n/a Signed-off-by: Joshua Marantz --- source/common/stats/utility.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/stats/utility.h b/source/common/stats/utility.h index 4328c2ef5875..dab8ba396c19 100644 --- a/source/common/stats/utility.h +++ b/source/common/stats/utility.h @@ -27,7 +27,7 @@ class DynamicName : public absl::string_view { // This is intentionally left as an implicit conversion from string_view to // make call-sites easier to read, e.g. // Utility::counterFromElements(*scope, {DynamicName("a"), DynamicName("b")}); - DynamicName(absl::string_view str) : absl::string_view(str) {} + explicit DynamicName(absl::string_view str) : absl::string_view(str) {} }; /** From 9f95dda44da9f7f5a3fc2c7a11500409189a0eba Mon Sep 17 00:00:00 2001 From: jianwen612 <55008549+jianwen612@users.noreply.github.com> Date: Fri, 14 Aug 2020 13:01:31 -0500 Subject: [PATCH 65/67] [fuzz]added an input check in writefilter fuzzer and added test cases (#12628) Added a handle for nullptr in HeaderPercentageProvider::percentage to avoid crash in mongo_proxy. Added many unit test cases into corpus so that the coverage can be improved. All those filters' coverage was increased by 20%-40%. Signed-off-by: jianwen --- .../filters/common/fault/fault_config.cc | 4 + .../dubbo_proxy_ondata_msg_split | 24 ++++ .../dubbo_proxy_ondata_twoway | 19 +++ .../dubbo_proxy_protocol_routing | 25 ++++ .../dubbo_proxy_protocol_routing_failure | 25 ++++ .../kafka_process_msg | 21 +++ .../network_readfilter_corpus/kafka_request1 | 17 +++ .../kafka_unknown_request | 21 +++ .../redis_proxy_1_auth_no_pwd_set | 24 ++++ .../redis_proxy_1_auth_pwd_set | 24 ++++ .../redis_proxy_1_bulk_string | 24 ++++ .../redis_proxy_1_negative_large_integer | 24 ++++ .../redis_proxy_1_nested_array | 24 ++++ .../redis_proxy_1_null | 24 ++++ .../rocketmq_proxy_end_stream | 22 +++ .../rocketmq_proxy_invalid_header | 15 +++ .../rocketmq_proxy_on_ack_msg | 15 +++ .../rocketmq_proxy_on_get_topic_route | 15 +++ .../rocketmq_proxy_on_heartbeat | 15 +++ .../rocketmq_proxy_on_pop_msg | 15 +++ .../rocketmq_proxy_sendmsg | 15 +++ .../rocketmq_proxy_sendmsg2 | 15 +++ .../rocketmq_proxy_unregistered_client | 16 +++ .../network_readfilter_corpus/thrift_proxy_1 | 13 ++ .../thrift_proxy_app_exception | 25 ++++ .../thrift_proxy_garbage_request | 25 ++++ .../thrift_proxy_invalid_msg_type | 25 ++++ .../thrift_proxy_on_data_handles_oneway | 20 +++ .../thrift_proxy_on_data_handles_thriftcall | 20 +++ .../thrift_proxy_pipelined_request1 | 20 +++ .../thrift_proxy_protocol_error | 25 ++++ .../thrift_proxy_router_test | 26 ++++ .../thrift_proxy_stop_and_resume | 25 ++++ .../zookeeper_proxy_auth | 17 +++ .../zookeeper_proxy_connect | 17 +++ .../zookeeper_proxy_multirequest | 17 +++ .../zookeeper_proxy_request_container | 17 +++ .../zookeeper_proxy_request_ephemeral | 17 +++ .../zookeeper_proxy_request_persistent | 17 +++ ...xy_request_persistent_ephemeral_sequential | 17 +++ ...keeper_proxy_request_persistent_sequential | 17 +++ .../zookeeper_proxy_request_ttl | 17 +++ .../zookeeper_proxy_request_ttl_sequential | 17 +++ .../zookeeper_proxy_watch_request | 17 +++ .../kafka_broker_process_response | 18 +++ .../kafka_broker_response1 | 18 +++ .../kafka_broker_unknown_response | 18 +++ .../mongodb_proxy_response | 55 ++++++++ .../mysql_proxy_msg_split | 125 ++++++++++++++++++ .../zookeeper_proxy_auth | 17 +++ .../zookeeper_proxy_connect | 17 +++ .../zookeeper_proxy_ping | 17 +++ .../zookeeper_proxy_watch_control | 17 +++ .../zookeeper_proxy_watch_event | 17 +++ 54 files changed, 1173 insertions(+) create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_msg_split create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_twoway create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing_failure create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_process_msg create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_request1 create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_unknown_request create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_no_pwd_set create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_pwd_set create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_bulk_string create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_negative_large_integer create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_nested_array create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_null create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_end_stream create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_invalid_header create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_ack_msg create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_get_topic_route create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_heartbeat create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_pop_msg create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg2 create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_unregistered_client create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_app_exception create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_garbage_request create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_invalid_msg_type create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_oneway create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_thriftcall create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_pipelined_request1 create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_protocol_error create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_router_test create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_stop_and_resume create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_auth create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_connect create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_multirequest create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_container create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ephemeral create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_ephemeral_sequential create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_sequential create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl_sequential create mode 100644 test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_watch_request create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_process_response create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_response1 create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_unknown_response create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_response create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_msg_split create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_auth create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_connect create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_ping create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_control create mode 100644 test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_event diff --git a/source/extensions/filters/common/fault/fault_config.cc b/source/extensions/filters/common/fault/fault_config.cc index ebbb86e2fd95..4053cc0b9006 100644 --- a/source/extensions/filters/common/fault/fault_config.cc +++ b/source/extensions/filters/common/fault/fault_config.cc @@ -13,6 +13,10 @@ namespace Fault { envoy::type::v3::FractionalPercent HeaderPercentageProvider::percentage(const Http::RequestHeaderMap* request_headers) const { + if (request_headers == nullptr) { + // If request_headers is nullptr, return the default percentage. + return percentage_; + } const auto header = request_headers->get(header_name_); if (header == nullptr) { return percentage_; diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_msg_split b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_msg_split new file mode 100644 index 000000000000..8560efc08450 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_msg_split @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.network.dubbo_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.dubbo_proxy.v3.DubboProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} +actions { + on_new_connection { + } +} + +actions { + on_data { + data: "\xda\xbb\xc2\x0\x0\x0\x0\x0\x0\x0\x0\xf\x0\x0\x0\x16\x5\x32\x2e\x30\x2e\x32" + } +} + +actions { + on_data { + data: "\x4\x74\x65\x73\x74\x5\x30\x2e\x30\x2e\x30\x4\x74\x65\x73\x74" + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_twoway b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_twoway new file mode 100644 index 000000000000..c61d2e7b6b03 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_ondata_twoway @@ -0,0 +1,19 @@ +config { + name: "envoy.filters.network.dubbo_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.dubbo_proxy.v3.DubboProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} +actions { + on_new_connection { + } +} + +actions { + on_data { + data: "\xda\xbb\xc2\x0\x0\x0\x0\x0\x0\x0\x0\xf\x0\x0\x0\x16\x5\x32\x2e\x30\x2e\x32\x4\x74\x65\x73\x74\x5\x30\x2e\x30\x2e\x30\x4\x74\x65\x73\x74" + } +} + + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing new file mode 100644 index 000000000000..bdffc0bf17cb --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing @@ -0,0 +1,25 @@ +config { + name: "envoy.filters.network.dubbo_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.dubbo_proxy.v3.DubboProxy" + value: "\xa\x4\x74\x65\x73\x74\x22\x5a\xa\x5\x74\x65\x73\x74\x31\x12\x21\x6f\x72\x67\x2e\x61\x70\x61\x63\x68\x65\x2e\x64\x75\x62\x62\x6f\x2e\x64\x65\x6d\x6f\x2e\x44\x65\x6d\x6f\x53\x65\x72\x76\x69\x63\x65\x2a\x2e\xa\xf\xa\xd\xa\xb\x2a\x9\xa\x0\x12\x5\x28\x2e\x2a\x3f\x29\x12\x1b\xa\x19\x75\x73\x65\x72\x5f\x73\x65\x72\x76\x69\x63\x65\x5f\x64\x75\x62\x62\x6f\x5f\x73\x65\x72\x76\x65\x72" + } +} +actions { + on_new_connection { + } +} + +actions { + on_data { + data: "\xda\xbb\xc2\x0\x0\x0\x0\x0\x0\x0\x0\x64\x0\x0\x0\x16\x5\x32\x2e\x30\x2e\x32\x4\x74\x65\x73\x74\x5\x30\x2e\x30\x2e\x30\x4\x74\x65\x73\x74" + } +} + +actions { + on_data { + data: "" + end_stream: true + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing_failure b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing_failure new file mode 100644 index 000000000000..8aedaa5377e3 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/dubbo_proxy_protocol_routing_failure @@ -0,0 +1,25 @@ +config { + name: "envoy.filters.network.dubbo_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.dubbo_proxy.v3.DubboProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} +actions { + on_new_connection { + } +} + +actions { + on_data { + data: "\xda\xbb\xc2\x0\x0\x0\x0\x0\x0\x0\x0\xf\x0\x0\x0\x16\x5\x32\x2e\x30\x2e\x32" + } +} + +actions { + on_data { + data: "" + end_stream: true + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_process_msg b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_process_msg new file mode 100644 index 000000000000..9c73c6e8c623 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_process_msg @@ -0,0 +1,21 @@ +config { + name: "envoy.filters.network.kafka_broker" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker" + value: "\n\"envoy.filters.network.kafka_broker" + } +} +actions { + on_new_connection { + } +} +actions { + on_data { + data:"\x0\x0\x0\x2e\x0\x0\x0\x0\x0\x0\x0\x0\x0\x2\x69\x64\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x2e\x0\x0\x0\x1\x0\x0\x0\x1\x0\x2\x69\x64\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x2e\x0\x0\x0\x2\x0\x0\x0\x2\x0\x2\x69\x64\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x36\x0\x0\x0\x3\x0\x0\x0\x3\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x36\x0\x0\x0\x4\x0\x0\x0\x4\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x36\x0\x0\x0\x5\x0\x0\x0\x5\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x36\x0\x0\x0\x6\x0\x0\x0\x6\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x36\x0\x0\x0\x7\x0\x0\x0\x7\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x36\x0\x0\x0\x8\x0\x0\x0\x8\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x38\x0\x1\x0\x0\x0\x0\x0\x9\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x38\x0\x1\x0\x1\x0\x0\x0\xa\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x38\x0\x1\x0\x2\x0\x0\x0\xb\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x3c\x0\x1\x0\x3\x0\x0\x0\xc\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x3d\x0\x1\x0\x4\x0\x0\x0\xd\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x45\x0\x1\x0\x5\x0\x0\x0\xe\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x45\x0\x1\x0\x6\x0\x0\x0\xf\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x61\x0\x1\x0\x7\x0\x0\x0\x10\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x61\x0\x1\x0\x8\x0\x0\x0\x11\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x65\x0\x1\x0\x9\x0\x0\x0\x12\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x65\x0\x1\x0\xa\x0\x0\x0\x13\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x6d\x0\x1\x0\xb\x0\x0\x0\x14\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x8\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x30\x0\x2\x0\x0\x0\x0\x0\x15\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x2c\x0\x2\x0\x1\x0\x0\x0\x16\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x2d\x0\x2\x0\x2\x0\x0\x0\x17\x0\x2\x69\x64\x0\x0\x0\x20\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x2d\x0\x2\x0\x3\x0\x0\x0\x18\x0\x2\x69\x64\x0\x0\x0\x20\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x31\x0\x2\x0\x4\x0\x0\x0\x19\x0\x2\x69\x64\x0\x0\x0\x20\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x31\x0\x2\x0\x5\x0\x0\x0\x1a\x0\x2\x69\x64\x0\x0\x0\x20\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x18\x0\x3\x0\x0\x0\x0\x0\x1b\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x18\x0\x3\x0\x1\x0\x0\x0\x1c\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x18\x0\x3\x0\x2\x0\x0\x0\x1d\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x18\x0\x3\x0\x3\x0\x0\x0\x1e\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x19\x0\x3\x0\x4\x0\x0\x0\x1f\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x19\x0\x3\x0\x5\x0\x0\x0\x20\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x19\x0\x3\x0\x6\x0\x0\x0\x21\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x19\x0\x3\x0\x7\x0\x0\x0\x22\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1b\x0\x3\x0\x8\x0\x0\x0\x23\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x2e\x0\x3\x0\x9\x0\x0\x0\x24\x0\x2\x69\x64\x0\x2\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x58\x0\x4\x0\x0\x0\x0\x0\x25\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x59\x0\x4\x0\x1\x0\x0\x0\x26\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x65\x0\x4\x0\x2\x0\x0\x0\x27\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x75\x0\x4\x0\x3\x0\x0\x0\x28\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x8b\x0\x4\x0\x4\x0\x0\x0\x29\x0\x2\x69\x64\x0\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x2\x0\x0\x0\x20\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x0\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x25\x0\x5\x0\x0\x0\x0\x0\x2a\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x31\x0\x5\x0\x1\x0\x0\x0\x2b\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x41\x0\x5\x0\x2\x0\x0\x0\x2c\x0\x2\x69\x64\x0\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x58\x0\x6\x0\x0\x0\x0\x0\x2d\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x5e\x0\x6\x0\x1\x0\x0\x0\x2e\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x66\x0\x6\x0\x2\x0\x0\x0\x2f\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x6e\x0\x6\x0\x3\x0\x0\x0\x30\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x76\x0\x6\x0\x4\x0\x0\x0\x31\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x82\x0\x6\x0\x5\x0\x0\x0\x32\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\xa1\x0\x6\x0\x6\x0\x0\x0\x33\x0\x2\x69\x64\x0\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x2\x0\x0\x0\x20\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x10\x0\x7\x0\x0\x0\x0\x0\x34\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x10\x0\x7\x0\x1\x0\x0\x0\x35\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x18\x0\x7\x0\x2\x0\x0\x0\x36\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x24\x0\x7\x0\x3\x0\x0\x0\x37\x0\x2\x69\x64\x0\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x38\x0\x8\x0\x0\x0\x0\x0\x38\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4c\x0\x8\x0\x1\x0\x0\x0\x39\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4c\x0\x8\x0\x2\x0\x0\x0\x3a\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4c\x0\x8\x0\x3\x0\x0\x0\x3b\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4c\x0\x8\x0\x4\x0\x0\x0\x3c\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x44\x0\x8\x0\x5\x0\x0\x0\x3d\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x48\x0\x8\x0\x6\x0\x0\x0\x3e\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x50\x0\x8\x0\x7\x0\x0\x0\x3f\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x67\x0\x8\x0\x8\x0\x0\x0\x40\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x28\x0\x9\x0\x0\x0\x0\x0\x41\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x28\x0\x9\x0\x1\x0\x0\x0\x42\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x28\x0\x9\x0\x2\x0\x0\x0\x43\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x28\x0\x9\x0\x3\x0\x0\x0\x44\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x28\x0\x9\x0\x4\x0\x0\x0\x45\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x28\x0\x9\x0\x5\x0\x0\x0\x46\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x37\x0\x9\x0\x6\x0\x0\x0\x47\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x14\x0\xa\x0\x0\x0\x0\x0\x48\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x15\x0\xa\x0\x1\x0\x0\x0\x49\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x0\x0\x15\x0\xa\x0\x2\x0\x0\x0\x4a\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x0\x0\x20\x0\xa\x0\x3\x0\x0\x0\x4b\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x8\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x3c\x0\xb\x0\x0\x0\x0\x0\x4c\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x40\x0\xb\x0\x1\x0\x0\x0\x4d\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x40\x0\xb\x0\x2\x0\x0\x0\x4e\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x40\x0\xb\x0\x3\x0\x0\x0\x4f\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x40\x0\xb\x0\x4\x0\x0\x0\x50\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x48\x0\xb\x0\x5\x0\x0\x0\x51\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x54\x0\xb\x0\x6\x0\x0\x0\x52\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x5\x0\x1\x2\x3\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x0\xc\x0\x0\x0\x0\x0\x53\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\xc\x0\x1\x0\x0\x0\x54\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\xc\x0\x2\x0\x0\x0\x55\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x28\x0\xc\x0\x3\x0\x0\x0\x56\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x31\x0\xc\x0\x4\x0\x0\x0\x57\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x1c\x0\xd\x0\x0\x0\x0\x0\x58\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1c\x0\xd\x0\x1\x0\x0\x0\x59\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1c\x0\xd\x0\x2\x0\x0\x0\x5a\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x28\x0\xd\x0\x3\x0\x0\x0\x5b\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x39\x0\xd\x0\x4\x0\x0\x0\x5c\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x34\x0\xe\x0\x0\x0\x0\x0\x5d\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x34\x0\xe\x0\x1\x0\x0\x0\x5e\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x34\x0\xe\x0\x2\x0\x0\x0\x5f\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x3c\x0\xe\x0\x3\x0\x0\x0\x60\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x49\x0\xe\x0\x4\x0\x0\x0\x61\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x5\x0\x1\x2\x3\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x18\x0\xf\x0\x0\x0\x0\x0\x62\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x18\x0\xf\x0\x1\x0\x0\x0\x63\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x18\x0\xf\x0\x2\x0\x0\x0\x64\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x19\x0\xf\x0\x3\x0\x0\x0\x65\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x19\x0\xf\x0\x4\x0\x0\x0\x66\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x21\x0\xf\x0\x5\x0\x0\x0\x67\x0\x2\x69\x64\x0\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\xc\x0\x10\x0\x0\x0\x0\x0\x68\x0\x2\x69\x64\x0\x0\x0\xc\x0\x10\x0\x1\x0\x0\x0\x69\x0\x2\x69\x64\x0\x0\x0\xc\x0\x10\x0\x2\x0\x0\x0\x6a\x0\x2\x69\x64\x0\x0\x0\x18\x0\x10\x0\x3\x0\x0\x0\x6b\x0\x2\x69\x64\x0\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x14\x0\x11\x0\x0\x0\x0\x0\x6c\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x14\x0\x11\x0\x1\x0\x0\x0\x6d\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\xc\x0\x12\x0\x0\x0\x0\x0\x6e\x0\x2\x69\x64\x0\x0\x0\xc\x0\x12\x0\x1\x0\x0\x0\x6f\x0\x2\x69\x64\x0\x0\x0\xc\x0\x12\x0\x2\x0\x0\x0\x70\x0\x2\x69\x64\x0\x0\x0\x26\x0\x12\x0\x3\x0\x0\x0\x71\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x46\x0\x13\x0\x0\x0\x0\x0\x72\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x47\x0\x13\x0\x1\x0\x0\x0\x73\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x0\x47\x0\x13\x0\x2\x0\x0\x0\x74\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x0\x47\x0\x13\x0\x3\x0\x0\x0\x75\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x0\x47\x0\x13\x0\x4\x0\x0\x0\x76\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x0\x65\x0\x13\x0\x5\x0\x0\x0\x77\x0\x2\x69\x64\x0\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x0\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x1c\x0\x14\x0\x0\x0\x0\x0\x78\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1c\x0\x14\x0\x1\x0\x0\x0\x79\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1c\x0\x14\x0\x2\x0\x0\x0\x7a\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1c\x0\x14\x0\x3\x0\x0\x0\x7b\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x24\x0\x14\x0\x4\x0\x0\x0\x7c\x0\x2\x69\x64\x0\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x2c\x0\x15\x0\x0\x0\x0\x0\x7d\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x2c\x0\x15\x0\x1\x0\x0\x0\x7e\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x18\x0\x16\x0\x0\x0\x0\x0\x7f\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x18\x0\x16\x0\x1\x0\x0\x0\x80\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x23\x0\x16\x0\x2\x0\x0\x0\x81\x0\x2\x69\x64\x0\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x24\x0\x17\x0\x0\x0\x0\x0\x82\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x24\x0\x17\x0\x1\x0\x0\x0\x83\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x28\x0\x17\x0\x2\x0\x0\x0\x84\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x2c\x0\x17\x0\x3\x0\x0\x0\x85\x0\x2\x69\x64\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x32\x0\x18\x0\x0\x0\x0\x0\x86\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x32\x0\x18\x0\x1\x0\x0\x0\x87\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x26\x0\x19\x0\x0\x0\x0\x0\x88\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x26\x0\x19\x0\x1\x0\x0\x0\x89\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1f\x0\x1a\x0\x0\x0\x0\x0\x8a\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x0\x1f\x0\x1a\x0\x1\x0\x0\x0\x8b\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x0\x33\x0\x1b\x0\x0\x0\x0\x0\x8c\x0\x2\x69\x64\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x4a\x0\x1c\x0\x0\x0\x0\x0\x8d\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4a\x0\x1c\x0\x1\x0\x0\x0\x8e\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4e\x0\x1c\x0\x2\x0\x0\x0\x8f\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x27\x0\x1d\x0\x0\x0\x0\x0\x90\x0\x2\x69\x64\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x28\x0\x1d\x0\x1\x0\x0\x0\x91\x0\x2\x69\x64\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x2b\x0\x1e\x0\x0\x0\x0\x0\x92\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x2c\x0\x1e\x0\x1\x0\x0\x0\x93\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x2b\x0\x1f\x0\x0\x0\x0\x0\x94\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x2c\x0\x1f\x0\x1\x0\x0\x0\x95\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x25\x0\x20\x0\x0\x0\x0\x0\x96\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x26\x0\x20\x0\x1\x0\x0\x0\x97\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x26\x0\x20\x0\x2\x0\x0\x0\x98\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x2e\x0\x21\x0\x0\x0\x0\x0\x99\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x2e\x0\x21\x0\x1\x0\x0\x0\x9a\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x2c\x0\x22\x0\x0\x0\x0\x0\x9b\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x2c\x0\x22\x0\x1\x0\x0\x0\x9c\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x23\x0\x0\x0\x0\x0\x9d\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x23\x0\x1\x0\x0\x0\x9e\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x14\x0\x24\x0\x0\x0\x0\x0\x9f\x0\x2\x69\x64\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x14\x0\x24\x0\x1\x0\x0\x0\xa0\x0\x2\x69\x64\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x2d\x0\x25\x0\x0\x0\x0\x0\xa1\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x2d\x0\x25\x0\x1\x0\x0\x0\xa2\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x28\x0\x26\x0\x0\x0\x0\x0\xa3\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x28\x0\x26\x0\x1\x0\x0\x0\xa4\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x3a\x0\x26\x0\x2\x0\x0\x0\xa5\x0\x2\x69\x64\x0\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x0\x0\x0\x0\x40\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x1c\x0\x27\x0\x0\x0\x0\x0\xa6\x0\x2\x69\x64\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1c\x0\x27\x0\x1\x0\x0\x0\xa7\x0\x2\x69\x64\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1c\x0\x28\x0\x0\x0\x0\x0\xa8\x0\x2\x69\x64\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1c\x0\x28\x0\x1\x0\x0\x0\xa9\x0\x2\x69\x64\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x29\x0\x0\x0\x0\x0\xaa\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x29\x0\x1\x0\x0\x0\xab\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x18\x0\x2a\x0\x0\x0\x0\x0\xac\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x18\x0\x2a\x0\x1\x0\x0\x0\xad\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x2a\x0\x2\x0\x0\x0\xae\x0\x2\x69\x64\x0\x2\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x24\x0\x2b\x0\x0\x0\x0\x0\xaf\x0\x2\x69\x64\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x25\x0\x2b\x0\x1\x0\x0\x0\xb0\x0\x2\x69\x64\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x35\x0\x2b\x0\x2\x0\x0\x0\xb1\x0\x2\x69\x64\x0\x8\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x2f\x0\x2c\x0\x0\x0\x0\x0\xb2\x0\x2\x69\x64\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x48\x0\x2c\x0\x1\x0\x0\x0\xb3\x0\x2\x69\x64\x0\x2\x8\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x8\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x44\x0\x2d\x0\x0\x0\x0\x0\xb4\x0\x2\x69\x64\x0\x0\x0\x0\x20\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x34\x0\x2e\x0\x0\x0\x0\x0\xb5\x0\x2\x69\x64\x0\x0\x0\x0\x20\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x28\x0\x2f\x0\x0\x0\x0\x0\xb6\x0\x2\x69\x64\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20" + } +} +actions { + advance_time { + milliseconds: 10000 + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_request1 b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_request1 new file mode 100644 index 000000000000..67bac6bc4f7f --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_request1 @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.kafka_broker" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker" + value: "\n\"envoy.filters.network.kafka_broker" + } +} +actions { + on_new_connection { + } +} +actions { + on_data { + data:"\x7f\xff\xff\xff\x0\x0\x0\x0\x0\x0\x0\x0\x80\x0" + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_unknown_request b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_unknown_request new file mode 100644 index 000000000000..f253ba9bfe58 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/kafka_unknown_request @@ -0,0 +1,21 @@ +config { + name: "envoy.filters.network.kafka_broker" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker" + value: "\n\"envoy.filters.network.kafka_broker" + } +} +actions { + on_new_connection { + } +} +actions { + on_data { + data:"\x0\x0\x0\x1d\x7f\xff\x0\x0\x0\x0\x0\x0\x0\x9\x63\x6c\x69\x65\x6e\x74\x2d\x69\x64\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0" + } +} +actions { + advance_time { + milliseconds: 10000 + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_no_pwd_set b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_no_pwd_set new file mode 100644 index 000000000000..1caceeb6b610 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_no_pwd_set @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.network.redis_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy" + value: "\n\001N\032\032\n\005\020\200\200\200\030\030\001 \377\377\377\337\017*\005\020\200\200\200\0302\000@\001*\010\n\006\032\004\001\000\000\010" + } +} +actions { + on_new_connection { + + } +} + +actions { + on_data { + data: "\x2a\x32\xd\xa\x24\x34\xd\xa\x61\x75\x74\x68\xd\xa\x24\x31\x32\xd\xa\x73\x6f\x6d\x65\x70\x61\x73\x73\x77\x6f\x72\x64\xd\xa" + } +} +actions { + on_data { + data: "0" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_pwd_set b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_pwd_set new file mode 100644 index 000000000000..ab9e717de323 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_auth_pwd_set @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.network.redis_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy" + value: "\xa\x61\x64\x6d\x69\x6e\x3a\xa\x20\x20\x61\x63\x63\x65\x73\x73\x5f\x6c\x6f\x67\x5f\x70\x61\x74\x68\x3a\x20\x2f\x64\x65\x76\x2f\x6e\x75\x6c\x6c\xa\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x73\x6f\x63\x6b\x65\x74\x5f\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\x20\x31\x32\x37\x2e\x30\x2e\x30\x2e\x31\xa\x20\x20\x20\x20\x20\x20\x70\x6f\x72\x74\x5f\x76\x61\x6c\x75\x65\x3a\x20\x30\xa\x73\x74\x61\x74\x69\x63\x5f\x72\x65\x73\x6f\x75\x72\x63\x65\x73\x3a\xa\x20\x20\x63\x6c\x75\x73\x74\x65\x72\x73\x3a\xa\x20\x20\x20\x20\x2d\x20\x6e\x61\x6d\x65\x3a\x20\x63\x6c\x75\x73\x74\x65\x72\x5f\x30\xa\x20\x20\x20\x20\x20\x20\x74\x79\x70\x65\x3a\x20\x53\x54\x41\x54\x49\x43\xa\x20\x20\x20\x20\x20\x20\x6c\x62\x5f\x70\x6f\x6c\x69\x63\x79\x3a\x20\x52\x41\x4e\x44\x4f\x4d\xa\x20\x20\x20\x20\x20\x20\x6c\x6f\x61\x64\x5f\x61\x73\x73\x69\x67\x6e\x6d\x65\x6e\x74\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x63\x6c\x75\x73\x74\x65\x72\x5f\x6e\x61\x6d\x65\x3a\x20\x63\x6c\x75\x73\x74\x65\x72\x5f\x30\xa\x20\x20\x20\x20\x20\x20\x20\x20\x65\x6e\x64\x70\x6f\x69\x6e\x74\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2d\x20\x6c\x62\x5f\x65\x6e\x64\x70\x6f\x69\x6e\x74\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2d\x20\x65\x6e\x64\x70\x6f\x69\x6e\x74\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x6f\x63\x6b\x65\x74\x5f\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\x20\x31\x32\x37\x2e\x30\x2e\x30\x2e\x31\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x70\x6f\x72\x74\x5f\x76\x61\x6c\x75\x65\x3a\x20\x30\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x2d\x20\x65\x6e\x64\x70\x6f\x69\x6e\x74\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x6f\x63\x6b\x65\x74\x5f\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\x20\x31\x32\x37\x2e\x30\x2e\x30\x2e\x31\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x70\x6f\x72\x74\x5f\x76\x61\x6c\x75\x65\x3a\x20\x30\xa\x20\x20\x6c\x69\x73\x74\x65\x6e\x65\x72\x73\x3a\xa\x20\x20\x20\x20\x6e\x61\x6d\x65\x3a\x20\x6c\x69\x73\x74\x65\x6e\x65\x72\x5f\x30\xa\x20\x20\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x73\x6f\x63\x6b\x65\x74\x5f\x61\x64\x64\x72\x65\x73\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x61\x64\x64\x72\x65\x73\x73\x3a\x20\x31\x32\x37\x2e\x30\x2e\x30\x2e\x31\xa\x20\x20\x20\x20\x20\x20\x20\x20\x70\x6f\x72\x74\x5f\x76\x61\x6c\x75\x65\x3a\x20\x30\xa\x20\x20\x20\x20\x66\x69\x6c\x74\x65\x72\x5f\x63\x68\x61\x69\x6e\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x66\x69\x6c\x74\x65\x72\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x6e\x61\x6d\x65\x3a\x20\x72\x65\x64\x69\x73\xa\x20\x20\x20\x20\x20\x20\x20\x20\x74\x79\x70\x65\x64\x5f\x63\x6f\x6e\x66\x69\x67\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x40\x74\x79\x70\x65\x22\x3a\x20\x74\x79\x70\x65\x2e\x67\x6f\x6f\x67\x6c\x65\x61\x70\x69\x73\x2e\x63\x6f\x6d\x2f\x65\x6e\x76\x6f\x79\x2e\x63\x6f\x6e\x66\x69\x67\x2e\x66\x69\x6c\x74\x65\x72\x2e\x6e\x65\x74\x77\x6f\x72\x6b\x2e\x72\x65\x64\x69\x73\x5f\x70\x72\x6f\x78\x79\x2e\x76\x32\x2e\x52\x65\x64\x69\x73\x50\x72\x6f\x78\x79\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x74\x61\x74\x5f\x70\x72\x65\x66\x69\x78\x3a\x20\x72\x65\x64\x69\x73\x5f\x73\x74\x61\x74\x73\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x70\x72\x65\x66\x69\x78\x5f\x72\x6f\x75\x74\x65\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x63\x61\x74\x63\x68\x5f\x61\x6c\x6c\x5f\x72\x6f\x75\x74\x65\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x63\x6c\x75\x73\x74\x65\x72\x3a\x20\x63\x6c\x75\x73\x74\x65\x72\x5f\x30\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x73\x65\x74\x74\x69\x6e\x67\x73\x3a\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6f\x70\x5f\x74\x69\x6d\x65\x6f\x75\x74\x3a\x20\x35\x73\xa\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x64\x6f\x77\x6e\x73\x74\x72\x65\x61\x6d\x5f\x61\x75\x74\x68\x5f\x70\x61\x73\x73\x77\x6f\x72\x64\x3a\x20\x7b\x20\x69\x6e\x6c\x69\x6e\x65\x5f\x73\x74\x72\x69\x6e\x67\x3a\x20\x73\x6f\x6d\x65\x70\x61\x73\x73\x77\x6f\x72\x64\x20\x7d\xa" + } +} +actions { + on_new_connection { + + } +} + +actions { + on_data { + data: "\x2a\x32\xd\xa\x24\x34\xd\xa\x61\x75\x74\x68\xd\xa\x24\x31\x32\xd\xa\x73\x6f\x6d\x65\x70\x61\x73\x73\x77\x6f\x72\x64\xd\xa" + } +} +actions { + on_data { + data: "0" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_bulk_string b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_bulk_string new file mode 100644 index 000000000000..2e52f9a217c2 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_bulk_string @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.network.redis_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy" + value: "\n\001N\032\032\n\005\020\200\200\200\030\030\001 \377\377\377\337\017*\005\020\200\200\200\0302\000@\001*\010\n\006\032\004\001\000\000\010" + } +} +actions { + on_new_connection { + + } +} + +actions { + on_data { + data: "\x24\x31\x31\xd\xa\x62\x75\x6c\x6b\x20\x73\x74\x72\x69\x6e\x67\xd\xa" + } +} +actions { + on_data { + data: "0" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_negative_large_integer b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_negative_large_integer new file mode 100644 index 000000000000..00a8b3bab8d6 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_negative_large_integer @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.network.redis_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy" + value: "\n\001N\032\032\n\005\020\200\200\200\030\030\001 \377\377\377\337\017*\005\020\200\200\200\0302\000@\001*\010\n\006\032\004\001\000\000\010" + } +} +actions { + on_new_connection { + + } +} + +actions { + on_data { + data: "\x2a\x32\xd\xa\x2a\x33\xd\xa\x24\x35\xd\xa\x68\x65\x6c\x6c\x6f\xd\xa\x3a\x30\xd\xa\x24\x2d\x31\xd\xa\x24\x35\xd\xa\x77\x6f\x72\x6c\x64\xd\xa" + } +} +actions { + on_data { + data: "0" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_nested_array b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_nested_array new file mode 100644 index 000000000000..00a8b3bab8d6 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_nested_array @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.network.redis_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy" + value: "\n\001N\032\032\n\005\020\200\200\200\030\030\001 \377\377\377\337\017*\005\020\200\200\200\0302\000@\001*\010\n\006\032\004\001\000\000\010" + } +} +actions { + on_new_connection { + + } +} + +actions { + on_data { + data: "\x2a\x32\xd\xa\x2a\x33\xd\xa\x24\x35\xd\xa\x68\x65\x6c\x6c\x6f\xd\xa\x3a\x30\xd\xa\x24\x2d\x31\xd\xa\x24\x35\xd\xa\x77\x6f\x72\x6c\x64\xd\xa" + } +} +actions { + on_data { + data: "0" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_null b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_null new file mode 100644 index 000000000000..9e1e03be28fa --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/redis_proxy_1_null @@ -0,0 +1,24 @@ +config { + name: "envoy.filters.network.redis_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.redis_proxy.v3.RedisProxy" + value: "\n\001N\032\032\n\005\020\200\200\200\030\030\001 \377\377\377\337\017*\005\020\200\200\200\0302\000@\001*\010\n\006\032\004\001\000\000\010" + } +} +actions { + on_new_connection { + + } +} + +actions { + on_data { + data: "\x24\x2d\x31\xd\xa" + } +} +actions { + on_data { + data: "0" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_end_stream b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_end_stream new file mode 100644 index 000000000000..0b7acac3be1c --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_end_stream @@ -0,0 +1,22 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x0\xa5\x0\x0\x0\xa1\x7b\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x63\x6f\x64\x65\x22\x3a\x33\x35\x2c\x22\x65\x78\x74\x46\x69\x65\x6c\x64\x73\x22\x3a\x7b\x22\x63\x6f\x6e\x73\x75\x6d\x65\x72\x47\x72\x6f\x75\x70\x22\x3a\x22\x74\x65\x73\x74\x5f\x63\x67\x22\x2c\x22\x63\x6c\x69\x65\x6e\x74\x49\x44\x22\x3a\x22\x74\x65\x73\x74\x5f\x63\x6c\x69\x65\x6e\x74\x5f\x69\x64\x22\x7d\x2c\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x37\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x2c\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x7d" + end_stream: yes + } +} + +actions { + on_data { + data: "" + end_stream: yes + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_invalid_header b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_invalid_header new file mode 100644 index 000000000000..4053e4278c2b --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_invalid_header @@ -0,0 +1,15 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x0\x1c\x1\x0\x0\x14\x72\x61\x6e\x64\x6f\x6d\x20\x74\x65\x78\x74\x20\x73\x75\x66\x66\x69\x63\x65\x73" + end_stream: false + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_ack_msg b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_ack_msg new file mode 100644 index 000000000000..8958ca2fe195 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_ack_msg @@ -0,0 +1,15 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x0\xcf\x0\x0\x0\xcb\x7b\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x63\x6f\x64\x65\x22\x3a\x35\x31\x2c\x22\x65\x78\x74\x46\x69\x65\x6c\x64\x73\x22\x3a\x7b\x22\x71\x75\x65\x75\x65\x49\x64\x22\x3a\x31\x2c\x22\x74\x6f\x70\x69\x63\x22\x3a\x22\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x22\x2c\x22\x6f\x66\x66\x73\x65\x74\x22\x3a\x31\x2c\x22\x63\x6f\x6e\x73\x75\x6d\x65\x72\x47\x72\x6f\x75\x70\x22\x3a\x22\x74\x65\x73\x74\x5f\x63\x67\x22\x2c\x22\x65\x78\x74\x72\x61\x49\x6e\x66\x6f\x22\x3a\x22\x74\x65\x73\x74\x5f\x65\x78\x74\x72\x61\x22\x7d\x2c\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x31\x38\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x7d" + end_stream: false + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_get_topic_route b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_get_topic_route new file mode 100644 index 000000000000..06b7e3544f95 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_get_topic_route @@ -0,0 +1,15 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x0\x86\x0\x0\x0\x82\x7b\x22\x65\x78\x74\x46\x69\x65\x6c\x64\x73\x22\x3a\x7b\x22\x74\x6f\x70\x69\x63\x22\x3a\x22\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x22\x7d\x2c\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x31\x33\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x2c\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x63\x6f\x64\x65\x22\x3a\x31\x30\x35\x7d" + end_stream: false + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_heartbeat b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_heartbeat new file mode 100644 index 000000000000..615e69219a98 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_heartbeat @@ -0,0 +1,15 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x4\x7a\x0\x0\x0\x5d\x7b\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x31\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x2c\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x63\x6f\x64\x65\x22\x3a\x33\x34\x7d\xa\x20\x20\x20\x20\x7b\xa\x20\x20\x20\x20\x20\x20\x22\x63\x6c\x69\x65\x6e\x74\x49\x44\x22\x3a\x20\x22\x31\x32\x37\x2e\x30\x2e\x30\x2e\x31\x40\x39\x30\x33\x33\x30\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x22\x63\x6f\x6e\x73\x75\x6d\x65\x72\x44\x61\x74\x61\x53\x65\x74\x22\x3a\x20\x5b\xa\x20\x20\x20\x20\x20\x20\x20\x20\x7b\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x63\x6f\x6e\x73\x75\x6d\x65\x46\x72\x6f\x6d\x57\x68\x65\x72\x65\x22\x3a\x20\x22\x43\x4f\x4e\x53\x55\x4d\x45\x5f\x46\x52\x4f\x4d\x5f\x46\x49\x52\x53\x54\x5f\x4f\x46\x46\x53\x45\x54\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x63\x6f\x6e\x73\x75\x6d\x65\x54\x79\x70\x65\x22\x3a\x20\x22\x43\x4f\x4e\x53\x55\x4d\x45\x5f\x50\x41\x53\x53\x49\x56\x45\x4c\x59\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x67\x72\x6f\x75\x70\x4e\x61\x6d\x65\x22\x3a\x20\x22\x74\x65\x73\x74\x5f\x63\x67\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x6d\x65\x73\x73\x61\x67\x65\x4d\x6f\x64\x65\x6c\x22\x3a\x20\x22\x43\x4c\x55\x53\x54\x45\x52\x49\x4e\x47\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x73\x75\x62\x73\x63\x72\x69\x70\x74\x69\x6f\x6e\x44\x61\x74\x61\x53\x65\x74\x22\x3a\x20\x5b\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x7b\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x63\x6c\x61\x73\x73\x46\x69\x6c\x74\x65\x72\x4d\x6f\x64\x65\x22\x3a\x20\x66\x61\x6c\x73\x65\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x63\x6f\x64\x65\x53\x65\x74\x22\x3a\x20\x5b\x5d\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x65\x78\x70\x72\x65\x73\x73\x69\x6f\x6e\x54\x79\x70\x65\x22\x3a\x20\x22\x54\x41\x47\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x73\x75\x62\x53\x74\x72\x69\x6e\x67\x22\x3a\x20\x22\x2a\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x73\x75\x62\x56\x65\x72\x73\x69\x6f\x6e\x22\x3a\x20\x31\x35\x37\x35\x36\x33\x30\x35\x38\x37\x39\x32\x35\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x74\x61\x67\x73\x53\x65\x74\x22\x3a\x20\x5b\x5d\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x74\x6f\x70\x69\x63\x22\x3a\x20\x22\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x22\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x7d\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x7b\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x63\x6c\x61\x73\x73\x46\x69\x6c\x74\x65\x72\x4d\x6f\x64\x65\x22\x3a\x20\x66\x61\x6c\x73\x65\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x63\x6f\x64\x65\x53\x65\x74\x22\x3a\x20\x5b\x5d\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x65\x78\x70\x72\x65\x73\x73\x69\x6f\x6e\x54\x79\x70\x65\x22\x3a\x20\x22\x54\x41\x47\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x73\x75\x62\x53\x74\x72\x69\x6e\x67\x22\x3a\x20\x22\x2a\x22\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x73\x75\x62\x56\x65\x72\x73\x69\x6f\x6e\x22\x3a\x20\x31\x35\x37\x35\x36\x33\x30\x35\x38\x37\x39\x34\x35\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x74\x61\x67\x73\x53\x65\x74\x22\x3a\x20\x5b\x5d\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x74\x6f\x70\x69\x63\x22\x3a\x20\x22\x25\x52\x45\x54\x52\x59\x25\x70\x6c\x65\x61\x73\x65\x5f\x72\x65\x6e\x61\x6d\x65\x5f\x75\x6e\x69\x71\x75\x65\x5f\x67\x72\x6f\x75\x70\x5f\x6e\x61\x6d\x65\x5f\x34\x22\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x7d\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x5d\x2c\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x75\x6e\x69\x74\x4d\x6f\x64\x65\x22\x3a\x20\x66\x61\x6c\x73\x65\xa\x20\x20\x20\x20\x20\x20\x20\x20\x7d\xa\x20\x20\x20\x20\x20\x20\x5d\x2c\xa\x20\x20\x20\x20\x20\x20\x22\x70\x72\x6f\x64\x75\x63\x65\x72\x44\x61\x74\x61\x53\x65\x74\x22\x3a\x20\x5b\xa\x20\x20\x20\x20\x20\x20\x20\x20\x7b\xa\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x22\x67\x72\x6f\x75\x70\x4e\x61\x6d\x65\x22\x3a\x20\x22\x43\x4c\x49\x45\x4e\x54\x5f\x49\x4e\x4e\x45\x52\x5f\x50\x52\x4f\x44\x55\x43\x45\x52\x22\xa\x20\x20\x20\x20\x20\x20\x20\x20\x7d\xa\x20\x20\x20\x20\x20\x20\x5d\xa\x20\x20\x20\x20\x7d\xa\x20\x20\x20\x20" + end_stream: false + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_pop_msg b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_pop_msg new file mode 100644 index 000000000000..34aa1101b423 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_on_pop_msg @@ -0,0 +1,15 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x0\xfd\x0\x0\x0\xf9\x7b\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x31\x37\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x2c\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x63\x6f\x64\x65\x22\x3a\x35\x30\x2c\x22\x65\x78\x74\x46\x69\x65\x6c\x64\x73\x22\x3a\x7b\x22\x62\x6f\x72\x6e\x54\x69\x6d\x65\x22\x3a\x31\x30\x30\x30\x2c\x22\x71\x75\x65\x75\x65\x49\x64\x22\x3a\x31\x2c\x22\x74\x6f\x70\x69\x63\x22\x3a\x22\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x22\x2c\x22\x69\x6e\x76\x69\x73\x69\x62\x6c\x65\x54\x69\x6d\x65\x22\x3a\x36\x30\x30\x30\x2c\x22\x63\x6f\x6e\x73\x75\x6d\x65\x72\x47\x72\x6f\x75\x70\x22\x3a\x22\x74\x65\x73\x74\x5f\x63\x67\x22\x2c\x22\x70\x6f\x6c\x6c\x54\x69\x6d\x65\x22\x3a\x33\x30\x30\x30\x2c\x22\x6d\x61\x78\x4d\x73\x67\x4e\x75\x6d\x73\x22\x3a\x33\x32\x2c\x22\x69\x6e\x69\x74\x4d\x6f\x64\x65\x22\x3a\x34\x7d\x7d" + end_stream: false + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg new file mode 100644 index 000000000000..1d38ed816210 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg @@ -0,0 +1,15 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x1\x9\x0\x0\x0\xf4\x7b\x22\x63\x6f\x64\x65\x22\x3a\x31\x30\x2c\x22\x65\x78\x74\x46\x69\x65\x6c\x64\x73\x22\x3a\x7b\x22\x73\x79\x73\x46\x6c\x61\x67\x22\x3a\x30\x2c\x22\x62\x6f\x72\x6e\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x22\x3a\x30\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x71\x75\x65\x75\x65\x49\x64\x22\x3a\x2d\x31\x2c\x22\x74\x6f\x70\x69\x63\x22\x3a\x22\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x22\x2c\x22\x64\x65\x66\x61\x75\x6c\x74\x54\x6f\x70\x69\x63\x22\x3a\x22\x22\x2c\x22\x70\x72\x6f\x64\x75\x63\x65\x72\x47\x72\x6f\x75\x70\x22\x3a\x22\x22\x2c\x22\x64\x65\x66\x61\x75\x6c\x74\x54\x6f\x70\x69\x63\x51\x75\x65\x75\x65\x4e\x75\x6d\x73\x22\x3a\x30\x7d\x2c\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x32\x30\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x2c\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x7d\x5f\x41\x70\x61\x63\x68\x65\x5f\x52\x6f\x63\x6b\x65\x74\x4d\x51\x5f" + end_stream: false + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg2 b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg2 new file mode 100644 index 000000000000..ae0810ea792e --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_sendmsg2 @@ -0,0 +1,15 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x0\xc7\x0\x0\x0\xb2\x7b\x22\x65\x78\x74\x46\x69\x65\x6c\x64\x73\x22\x3a\x7b\x22\x61\x22\x3a\x22\x74\x65\x73\x74\x5f\x70\x67\x22\x2c\x22\x62\x22\x3a\x22\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x22\x2c\x22\x63\x22\x3a\x22\x22\x2c\x22\x64\x22\x3a\x30\x2c\x22\x65\x22\x3a\x2d\x31\x2c\x22\x66\x22\x3a\x30\x2c\x22\x67\x22\x3a\x30\x2c\x22\x68\x22\x3a\x30\x7d\x2c\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x32\x31\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x2c\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x63\x6f\x64\x65\x22\x3a\x33\x31\x30\x7d\x5f\x41\x70\x61\x63\x68\x65\x5f\x52\x6f\x63\x6b\x65\x74\x4d\x51\x5f" + end_stream: false + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_unregistered_client b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_unregistered_client new file mode 100644 index 000000000000..617d3c290b72 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/rocketmq_proxy_unregistered_client @@ -0,0 +1,16 @@ +config { + name: "envoy.filters.network.rocketmq_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.rocketmq_proxy.v3.RocketmqProxy" + value: "\xa\x4\x74\x65\x73\x74\x12\x31\xa\xd\x64\x65\x66\x61\x75\x6c\x74\x5f\x72\x6f\x75\x74\x65\x12\x20\xa\xe\xa\xc\xa\xa\x74\x65\x73\x74\x5f\x74\x6f\x70\x69\x63\x12\xe\xa\xc\x66\x61\x6b\x65\x5f\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_data { + data: "\x0\x0\x0\xa5\x0\x0\x0\xa1\x7b\x22\x65\x78\x74\x46\x69\x65\x6c\x64\x73\x22\x3a\x7b\x22\x63\x6c\x69\x65\x6e\x74\x49\x44\x22\x3a\x22\x74\x65\x73\x74\x5f\x63\x6c\x69\x65\x6e\x74\x5f\x69\x64\x22\x2c\x22\x63\x6f\x6e\x73\x75\x6d\x65\x72\x47\x72\x6f\x75\x70\x22\x3a\x22\x74\x65\x73\x74\x5f\x63\x67\x22\x7d\x2c\x22\x6f\x70\x61\x71\x75\x65\x22\x3a\x37\x2c\x22\x73\x65\x72\x69\x61\x6c\x69\x7a\x65\x54\x79\x70\x65\x43\x75\x72\x72\x65\x6e\x74\x52\x50\x43\x22\x3a\x22\x4a\x53\x4f\x4e\x22\x2c\x22\x76\x65\x72\x73\x69\x6f\x6e\x22\x3a\x30\x2c\x22\x6c\x61\x6e\x67\x75\x61\x67\x65\x22\x3a\x22\x43\x50\x50\x22\x2c\x22\x66\x6c\x61\x67\x22\x3a\x30\x2c\x22\x63\x6f\x64\x65\x22\x3a\x33\x35\x7d" + end_stream: false + } +} + + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_1 b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_1 index a194b7f99031..f7fec0a7c7ba 100644 --- a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_1 +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_1 @@ -5,3 +5,16 @@ config { value: "\nYtype.googleapis.com/envoy.extensions.filters.network.thrift_proxy.vLLLLLLLLL3.ThriftProxy\020\003\030\003\"\231\002\022\226\002\n\003\n\001A\022\216\002\032\201\002\n\361\001\n\010@\000\000\000\000\000\000\000\022\344\001\nc\n_*]\032[\nPtype.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy\022\007\020\002\"\003\n\001A\022\000\n}\nyenvoy.filters.network.thrift_prox\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177y\022\000\n\013\n\000\022\007\n\005\n\001#\022\0002\010A\177\177\177\177\177\177\177" } } + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\0\0\0\144\17\377\0\0\0\0\0\1\0\1\0\2\1\2\0\0" + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_app_exception b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_app_exception new file mode 100644 index 000000000000..b9f4e7a02456 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_app_exception @@ -0,0 +1,25 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x64\xf\xff\x0\x0\x0\x0\x0\x1\x0\x1\x0\x2\x1\x2\x0\x0" + } +} + +actions { + on_data { + data: "" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_garbage_request b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_garbage_request new file mode 100644 index 000000000000..4109fe67ba90 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_garbage_request @@ -0,0 +1,25 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x0\x0\x0\x0\x0" + } +} + +actions { + on_data { + data: "" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_invalid_msg_type b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_invalid_msg_type new file mode 100644 index 000000000000..802d56801cbf --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_invalid_msg_type @@ -0,0 +1,25 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x1f\x80\x1\x0\x1\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\x1\x8\xff\xff" + } +} + +actions { + on_data { + data: "" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_oneway b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_oneway new file mode 100644 index 000000000000..967027b75367 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_oneway @@ -0,0 +1,20 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x1d\x80\x1\x0\x4\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\xf\xb\x0\x0\x0\x0\x0\x5\x66\x69\x65\x6c\x64\x0" + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_thriftcall b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_thriftcall new file mode 100644 index 000000000000..69e2926e9014 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_on_data_handles_thriftcall @@ -0,0 +1,20 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x1d\x80\x1\x0\x1\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\xf\xb\x0\x0\x0\x0\x0\x5\x66\x69\x65\x6c\x64\x0" + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_pipelined_request1 b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_pipelined_request1 new file mode 100644 index 000000000000..81062c6c8f1b --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_pipelined_request1 @@ -0,0 +1,20 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x1d\x80\x1\x0\x1\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\x1\xb\x0\x0\x0\x0\x0\x5\x66\x69\x65\x6c\x64\x0\x0\x0\x0\x1d\x80\x1\x0\x1\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\x2\xb\x0\x0\x0\x0\x0\x5\x66\x69\x65\x6c\x64\x0" + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_protocol_error b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_protocol_error new file mode 100644 index 000000000000..82900253616b --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_protocol_error @@ -0,0 +1,25 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x1d\x80\x1\x0\xff\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\x1\x0" + } +} + +actions { + on_data { + data: "" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_router_test b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_router_test new file mode 100644 index 000000000000..dd30ffe54e37 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_router_test @@ -0,0 +1,26 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74\x10\x1\x18\x1\x22\x1d\xa\x6\x72\x6f\x75\x74\x65\x73\x12\x13\xa\x6\xa\x4\x6e\x61\x6d\x65\x12\x9\xa\x7\x63\x6c\x75\x73\x74\x65\x72" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x1d\x80\x1\x0\x4\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\xf\xb\x0\x0\x0\x0\x0\x5\x66\x69\x65\x6c\x64\x0" + } +} + +actions { + on_data { + data: "" + end_stream: true + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_stop_and_resume b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_stop_and_resume new file mode 100644 index 000000000000..aecd8c383fd8 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/thrift_proxy_stop_and_resume @@ -0,0 +1,25 @@ +config { + name: "envoy.filters.network.thrift_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy" + value: "\xa\x4\x74\x65\x73\x74" + } +} + +actions { + on_new_connection { + } +} + + +actions { + on_data { + data: "\x0\x0\x0\x1d\x80\x1\x0\x4\x0\x0\x0\x4\x6e\x61\x6d\x65\x0\x0\x0\xf\xb\x0\x0\x0\x0\x0\x5\x66\x69\x65\x6c\x64\x0" + } +} + +actions { + on_data { + data: "" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_auth b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_auth new file mode 100644 index 000000000000..a266ee5c9f87 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_auth @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x28\xff\xff\xff\xfc\x0\x0\x0\x64\x0\x0\x0\x0\x0\x0\x0\x6\x64\x69\x67\x65\x73\x74\x0\x0\x0\x6\x70\x40\x73\x73\x77\x64" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_connect b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_connect new file mode 100644 index 000000000000..de17bd097db3 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_connect @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1c\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x64\x0\x0\x0\xa\x0\x0\x0\x0\x0\x0\x0\xc8\x0\x0\x0\x0" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_multirequest b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_multirequest new file mode 100644 index 000000000000..2fe0833924df --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_multirequest @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x7c\x0\x0\x3\xe8\x0\x0\x0\xe\x0\x0\x0\x1\x0\xff\xff\xff\xff\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x1\x31\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1\x0\xff\xff\xff\xff\x0\x0\x0\x4\x2f\x62\x61\x72\x0\x0\x0\x1\x31\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\xd\x0\xff\xff\xff\xff\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x64\x0\x0\x0\x5\x0\xff\xff\xff\xff\x0\x0\x0\x4\x2f\x62\x61\x72\x0\x0\x0\x1\x32\xff\xff\xff\xff\xff\xff\xff\xff\x1\xff\xff\xff\xff" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_container b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_container new file mode 100644 index 000000000000..65fb75866297 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_container @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1f\x0\x0\x3\xe8\x0\x0\x0\x13\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x3\x62\x61\x72\x0\x0\x0\x0\x0\x0\x0\x4" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ephemeral b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ephemeral new file mode 100644 index 000000000000..c9b32803a8ed --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ephemeral @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1f\x0\x0\x3\xe8\x0\x0\x0\x1\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x3\x62\x61\x72\x0\x0\x0\x0\x0\x0\x0\x1" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent new file mode 100644 index 000000000000..1f01b8502432 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1f\x0\x0\x3\xe8\x0\x0\x0\x1\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x3\x62\x61\x72\x0\x0\x0\x0\x0\x0\x0\x0" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_ephemeral_sequential b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_ephemeral_sequential new file mode 100644 index 000000000000..91fd261faf9a --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_ephemeral_sequential @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1f\x0\x0\x3\xe8\x0\x0\x0\x1\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x3\x62\x61\x72\x0\x0\x0\x0\x0\x0\x0\x3" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_sequential b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_sequential new file mode 100644 index 000000000000..c9b32803a8ed --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_persistent_sequential @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1f\x0\x0\x3\xe8\x0\x0\x0\x1\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x3\x62\x61\x72\x0\x0\x0\x0\x0\x0\x0\x1" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl new file mode 100644 index 000000000000..dabe94508782 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1f\x0\x0\x3\xe8\x0\x0\x0\x15\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x3\x62\x61\x72\x0\x0\x0\x0\x0\x0\x0\x5" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl_sequential b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl_sequential new file mode 100644 index 000000000000..24d933cf2f98 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_request_ttl_sequential @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x1f\x0\x0\x3\xe8\x0\x0\x0\x15\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x3\x62\x61\x72\x0\x0\x0\x0\x0\x0\x0\x6" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_watch_request b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_watch_request new file mode 100644 index 000000000000..3c46ff026522 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_readfilter_corpus/zookeeper_proxy_watch_request @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_data { + data: "\x0\x0\x0\x48\xff\xff\xff\xf8\x0\x0\x0\x65\x0\x0\x0\x2\x0\x0\x0\x4\x2f\x66\x6f\x6f\x0\x0\x0\x4\x2f\x62\x61\x72\x0\x0\x0\x2\x0\x0\x0\x5\x2f\x66\x6f\x6f\x31\x0\x0\x0\x5\x2f\x62\x61\x72\x31\x0\x0\x0\x2\x0\x0\x0\x5\x2f\x66\x6f\x6f\x32\x0\x0\x0\x5\x2f\x62\x61\x72\x32" + } +} +actions { + on_data { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_process_response b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_process_response new file mode 100644 index 000000000000..5f5f815bcf1e --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_process_response @@ -0,0 +1,18 @@ +config { + name: "envoy.filters.network.kafka_broker" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker" + value: "\n}\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } +} +actions { + on_write { + data: "\x0\x0\x0\x22\x0\x0\x0\x0\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x26\x0\x0\x0\x1\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x2e\x0\x0\x0\x2\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x2e\x0\x0\x0\x3\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x2e\x0\x0\x0\x4\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x36\x0\x0\x0\x5\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x36\x0\x0\x0\x6\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x36\x0\x0\x0\x7\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x4e\x0\x0\x0\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x2a\x0\x0\x0\x9\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x2e\x0\x0\x0\xa\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x2e\x0\x0\x0\xb\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x2e\x0\x0\x0\xc\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x4a\x0\x0\x0\xd\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x52\x0\x0\x0\xe\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x52\x0\x0\x0\xf\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x58\x0\x0\x0\x10\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x58\x0\x0\x0\x11\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x58\x0\x0\x0\x12\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x58\x0\x0\x0\x13\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x5c\x0\x0\x0\x14\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x26\x0\x0\x0\x15\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x2a\x0\x0\x0\x16\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x2e\x0\x0\x0\x17\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x2e\x0\x0\x0\x18\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x32\x0\x0\x0\x19\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x32\x0\x0\x0\x1a\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x44\x0\x0\x0\x1b\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x51\x0\x0\x0\x1c\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x59\x0\x0\x0\x1d\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x5d\x0\x0\x0\x1e\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x5d\x0\x0\x0\x1f\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x65\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x65\x0\x0\x0\x21\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x69\x0\x0\x0\x22\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x71\x0\x0\x0\x23\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x88\x0\x0\x0\x24\x0\x0\x0\x0\x20\x2\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x2\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x0\x2\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x18\x0\x0\x0\x25\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x18\x0\x0\x0\x26\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x18\x0\x0\x0\x27\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x18\x0\x0\x0\x28\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x2b\x0\x0\x0\x29\x0\x0\x10\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x18\x0\x0\x0\x2a\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x18\x0\x0\x0\x2b\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x0\x0\x0\x2b\x0\x0\x0\x2c\x0\x0\x10\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x6\x0\x0\x0\x2d\x0\x10\x0\x0\x0\x6\x0\x0\x0\x2e\x0\x10\x0\x0\x0\x6\x0\x0\x0\x2f\x0\x10\x0\x0\x0\x6\x0\x0\x0\x30\x0\x10\x0\x0\x0\x6\x0\x0\x0\x31\x0\x10\x0\x0\x0\x6\x0\x0\x0\x32\x0\x10\x0\x0\x0\x12\x0\x0\x0\x33\x0\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x16\x0\x0\x0\x34\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x16\x0\x0\x0\x35\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x16\x0\x0\x0\x36\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x29\x0\x0\x0\x37\x0\x0\x10\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x1a\x0\x0\x0\x38\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1a\x0\x0\x0\x39\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1a\x0\x0\x0\x3a\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x3b\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x3c\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x3d\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x3e\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x3f\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x39\x0\x0\x0\x40\x0\x0\x0\x0\x20\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x2a\x0\x0\x0\x41\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x2a\x0\x0\x0\x42\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x2c\x0\x0\x0\x43\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x10\x0\x0\x0\x30\x0\x0\x0\x44\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x10\x0\x0\x0\x30\x0\x0\x0\x45\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x10\x0\x0\x0\x34\x0\x0\x0\x46\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x10\x0\x0\x0\x4e\x0\x0\x0\x47\x0\x0\x0\x0\x20\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x16\x0\x0\x0\x48\x0\x10\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x22\x0\x0\x0\x49\x0\x0\x0\x20\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x22\x0\x0\x0\x4a\x0\x0\x0\x20\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x2c\x0\x0\x0\x4b\x0\x0\x0\x0\x20\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x36\x0\x0\x0\x4c\x0\x10\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x36\x0\x0\x0\x4d\x0\x10\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x3a\x0\x0\x0\x4e\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x3a\x0\x0\x0\x4f\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x3a\x0\x0\x0\x50\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x42\x0\x0\x0\x51\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x4e\x0\x0\x0\x52\x0\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x5\x0\x1\x2\x3\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x6\x0\x0\x0\x53\x0\x10\x0\x0\x0\xa\x0\x0\x0\x54\x0\x0\x0\x20\x0\x10\x0\x0\x0\xa\x0\x0\x0\x55\x0\x0\x0\x20\x0\x10\x0\x0\x0\xa\x0\x0\x0\x56\x0\x0\x0\x20\x0\x10\x0\x0\x0\x16\x0\x0\x0\x57\x0\x0\x0\x0\x20\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x6\x0\x0\x0\x58\x0\x10\x0\x0\x0\xa\x0\x0\x0\x59\x0\x0\x0\x20\x0\x10\x0\x0\x0\xa\x0\x0\x0\x5a\x0\x0\x0\x20\x0\x10\x0\x0\x0\x20\x0\x0\x0\x5b\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x32\x0\x0\x0\x5c\x0\x0\x0\x0\x20\x0\x10\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\xe\x0\x0\x0\x5d\x0\x10\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x12\x0\x0\x0\x5e\x0\x0\x0\x20\x0\x10\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x12\x0\x0\x0\x5f\x0\x0\x0\x20\x0\x10\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x12\x0\x0\x0\x60\x0\x0\x0\x20\x0\x10\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x1b\x0\x0\x0\x61\x0\x0\x0\x0\x20\x0\x10\x5\x0\x1\x2\x3\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x56\x0\x0\x0\x62\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x5a\x0\x0\x0\x63\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x5a\x0\x0\x0\x64\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x5e\x0\x0\x0\x65\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x20\x0\x0\x0\x66\x0\x0\x0\x66\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x20\x0\x0\x0\x74\x0\x0\x0\x67\x0\x0\x0\x0\x20\x2\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x5\x0\x1\x2\x3\x5\x0\x1\x2\x3\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x1a\x0\x0\x0\x68\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1e\x0\x0\x0\x69\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1e\x0\x0\x0\x6a\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x30\x0\x0\x0\x6b\x0\x0\x0\x0\x20\x0\x10\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x12\x0\x0\x0\x6c\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x12\x0\x0\x0\x6d\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x10\x0\x0\x0\x6e\x0\x10\x0\x0\x0\x1\x0\x10\x0\x10\x0\x10\x0\x0\x0\x14\x0\x0\x0\x6f\x0\x10\x0\x0\x0\x1\x0\x10\x0\x10\x0\x10\x0\x0\x0\x20\x0\x0\x0\x14\x0\x0\x0\x70\x0\x10\x0\x0\x0\x1\x0\x10\x0\x10\x0\x10\x0\x0\x0\x20\x0\x0\x0\x27\x0\x0\x0\x71\x0\x10\x2\x0\x10\x0\x10\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x12\x0\x0\x0\x72\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x1a\x0\x0\x0\x73\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1e\x0\x0\x0\x74\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1e\x0\x0\x0\x75\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1e\x0\x0\x0\x76\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x53\x0\x0\x0\x77\x0\x0\x0\x0\x20\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x10\x2\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x0\x8\x0\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x12\x0\x0\x0\x78\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x16\x0\x0\x0\x79\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x16\x0\x0\x0\x7a\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x16\x0\x0\x0\x7b\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x29\x0\x0\x0\x7c\x0\x0\x0\x0\x20\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x26\x0\x0\x0\x7d\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x26\x0\x0\x0\x7e\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x14\x0\x0\x0\x7f\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x14\x0\x0\x0\x80\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x0\x0\x0\x20\x0\x0\x0\x81\x0\x0\x0\x0\x20\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x22\x0\x0\x0\x82\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x26\x0\x0\x0\x83\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x2a\x0\x0\x0\x84\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x2a\x0\x0\x0\x85\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x10\x0\x0\x0\x20\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1e\x0\x0\x0\x86\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x87\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\xa\x0\x0\x0\x88\x0\x0\x0\x20\x0\x10\x0\x0\x0\xa\x0\x0\x0\x89\x0\x0\x0\x20\x0\x10\x0\x0\x0\xa\x0\x0\x0\x8a\x0\x0\x0\x20\x0\x10\x0\x0\x0\xa\x0\x0\x0\x8b\x0\x0\x0\x20\x0\x10\x0\x0\x0\x26\x0\x0\x0\x8c\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x8d\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x8e\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x8f\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x35\x0\x0\x0\x90\x0\x0\x0\x20\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x36\x0\x0\x0\x91\x0\x0\x0\x20\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x16\x0\x0\x0\x92\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x16\x0\x0\x0\x93\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x3f\x0\x0\x0\x94\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x40\x0\x0\x0\x95\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x8\x0\x0\x0\x36\x0\x0\x0\x96\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x4b\x0\x0\x0\x97\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x8\x0\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x0\x0\x4b\x0\x0\x0\x98\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x8\x0\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x0\x0\x1f\x0\x0\x0\x99\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1f\x0\x0\x0\x9a\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1e\x0\x0\x0\x9b\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1e\x0\x0\x0\x9c\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x0\x0\x3b\x0\x0\x0\x9d\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x3b\x0\x0\x0\x9e\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x16\x0\x0\x0\x9f\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x1e\x0\x0\x0\xa0\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x1e\x0\x0\x0\xa1\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1e\x0\x0\x0\xa2\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x42\x0\x0\x0\xa3\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x20\x0\x0\x0\x42\x0\x0\x0\xa4\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x20\x0\x0\x0\x48\x0\x0\x0\xa5\x0\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x7\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x7\x73\x74\x72\x69\x6e\x67\x5\x0\x1\x2\x3\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x12\x0\x0\x0\xa6\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x12\x0\x0\x0\xa7\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x12\x0\x0\x0\xa8\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x12\x0\x0\x0\xa9\x0\x10\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x20\x0\x0\x0\x5a\x0\x0\x0\xaa\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x5a\x0\x0\x0\xab\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x0\x0\x0\x0\x0\x0\x40\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x4\x0\x1\x2\x3\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x20\x0\x0\x0\x16\x0\x0\x0\xac\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x16\x0\x0\x0\xad\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x10\x0\x0\x0\x29\x0\x0\x0\xae\x0\x0\x0\x0\x20\x2\x7\x73\x74\x72\x69\x6e\x67\x0\x10\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x26\x0\x0\x0\xaf\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x28\x0\x0\x0\xb0\x0\x0\x0\x20\x0\x10\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x42\x0\x0\x0\xb1\x0\x0\x0\x0\x20\x0\x10\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x1f\x0\x0\x0\xb2\x0\x0\x0\x20\x0\x0\x0\x1\x0\x10\x0\x6\x73\x74\x72\x69\x6e\x67\x8\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x31\x0\x0\x0\xb3\x0\x0\x0\x0\x20\x2\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x8\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x49\x0\x0\x0\xb4\x0\x0\x0\x0\x20\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x4f\x0\x0\x0\xb5\x0\x0\x0\x0\x20\x0\x10\x7\x73\x74\x72\x69\x6e\x67\x2\x7\x73\x74\x72\x69\x6e\x67\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\x0\x0\x0\x20\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x2\xa\x3\x1\x2\x3\x14\x3\x4\x5\x6\x0\x0\x0\x20\x0\x0\x0\xb6\x0\x10\x0\x0\x0\x20\x0\x0\x0\x1\x0\x6\x73\x74\x72\x69\x6e\x67\x0\x0\x0\x1\x0\x0\x0\x20\x0\x10" + } +} +actions { + on_write { + data: "" + end_stream: true + } +} \ No newline at end of file diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_response1 b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_response1 new file mode 100644 index 000000000000..bf2ad07c74c8 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_response1 @@ -0,0 +1,18 @@ +config { + name: "envoy.filters.network.kafka_broker" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker" + value: "\n}\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } +} +actions { + on_write { + data: "\x7f\xff\xff\xff\x0\x0\x0\x2a\x80\x0\x0\x0" + } +} +actions { + on_write { + data: "" + end_stream: true + } +} \ No newline at end of file diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_unknown_response b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_unknown_response new file mode 100644 index 000000000000..9cdb94e228d7 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/kafka_broker_unknown_response @@ -0,0 +1,18 @@ +config { + name: "envoy.filters.network.kafka_broker" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.kafka_broker.v3.KafkaBroker" + value: "\n}\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } +} +actions { + on_write { + data: "\x0\x0\x0\x8\x0\x0\x0\x0\x0\x0\x0\x0" + } +} +actions { + on_write { + data: "" + end_stream: true + } +} \ No newline at end of file diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_response b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_response new file mode 100644 index 000000000000..be400868606f --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mongodb_proxy_response @@ -0,0 +1,55 @@ +config { + name: "envoy.filters.network.mongo_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.mongo_proxy.v3.MongoProxy" + value: "\xa\x4\x74\x65\x73\x74\x1a\x4\x1a\x2\x8\x1" + } +} + +actions { + on_write { + data: "\120\0\0\0\1\0\0\0\1\0\0\0\324\7\0\0\4\0\0\0\164\145\163\164\56\164\145\163\164\0\24\0\0\0\377\377\377\377\52\0\0\0\2\163\164\162\151\156\147\137\156\145\145\144\137\145\163\143\0\20\0\0\0\173\42\146\157\157\42\72\40\42\142\141\162\12\42\175\0\0" + } +} + +actions { + on_write { + data: "\56\0\0\0\2\0\0\0\2\0\0\0\1\0\0\0\10\0\0\0\40\116\0\0\0\0\0\0\24\0\0\0\2\0\0\0\5\0\0\0\0\5\0\0\0\0" + } +} + + +actions { + on_write { + data: "\45\0\0\0\3\0\0\0\3\0\0\0\325\7\0\0\0\0\0\0\164\145\163\164\0\24\0\0\0\40\116\0\0\0\0\0\0" + } +} + +actions { + on_write { + data: "\43\0\0\0\4\0\0\0\4\0\0\0\322\7\0\0\10\0\0\0\164\145\163\164\0\5\0\0\0\0\5\0\0\0\0" + } +} + + + + +actions { + on_write { + data: "\50\0\0\0\5\0\0\0\5\0\0\0\327\7\0\0\0\0\0\0\2\0\0\0\40\116\0\0\0\0\0\0\100\234\0\0\0\0\0\0" + } +} + + + +actions { + on_write { + data: "\120\0\0\0\17\0\0\0\31\0\0\0\332\7\0\0\124\145\163\164\40\144\141\164\141\142\141\163\145\0\124\145\163\164\40\143\157\155\155\141\156\144\40\156\141\155\145\0\5\0\0\0\0\5\0\0\0\0\26\0\0\0\2\167\157\162\154\144\0\6\0\0\0\150\145\154\154\157\0\0" + } +} + +actions { + on_write { + data: "\60\0\0\0\20\0\0\0\32\0\0\0\333\7\0\0\5\0\0\0\0\5\0\0\0\0\26\0\0\0\2\167\157\162\154\144\0\6\0\0\0\150\145\154\154\157\0\0" + } +} diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_msg_split b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_msg_split new file mode 100644 index 000000000000..f95b3684b202 --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/mysql_proxy_msg_split @@ -0,0 +1,125 @@ +config { + name: "envoy.filters.network.mysql_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.mysql_proxy.v3.MySQLProxy" + value: "\n\006#\336\215\302\246\001" + } +} + { + on_write { + data: "\34\0\0\0\12\65\56\60\56\65\64\0\136\0\0\0\41\100\163\141\154\164\43\44\0\1\1\41\0\2\0\2" + } +} + +actions { + on_write { + data: "\57\0\0\1\0\0\0\3\1\0\0\0\41\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\165\163\145\162\61\0\160\64\44\44\167\60\162\66\0" + } +} + +actions { + on_write { + data: "\7\0\0\2\376\1\0\0\0\1\0" + } +} + +actions { + on_write { + data: "\14\0\0\3\155\171\163\161\154\137\157\160\141\161\165\145" + } +} + +actions { + on_write { + data: "\7\0\0\4\377\1\0\0\0\1\0" + } +} + +actions { + on_write { + data: "\30\0\0\0\3\103\122\105\101\124\105\40\104\101\124\101\102\101\123\105\40\155\171\163\161\154\144\142" + } +} + +actions { + on_write { + data: "\34\0\0\0\12\65\56\60\56\65\64\0\136\0\0\0\41\100\163\141\154\164\43\44\0\1\1\41\0\2\0\2" + } +} +actions { + advance_time { + milliseconds: 14848 + } +} +actions { + on_write { + data: "\7\0\0\2\377\1\0\0\0\1\0" + end_stream: true + } +} +actions { + on_new_connection { + } +} +actions { + on_write { + data: "\57\0\0\1\0\2\0\3\1\0\0\0\41\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\165\163\145\162\61\0\160\64\44\44\167\60\162\66\0" + } +} +actions { + on_write { + data: "\7\0\0\2\0\1\0\0\0\1\0" + } +} +actions { + on_write { + data: "\7\0\0\2\376\1\0\0\0\1\0" + end_stream: true + } +} +actions { + on_write { + data: "\7\0\0\4\377\1\0\0\0\1\0" + } +} +actions { + on_write { + data: "\57\0\0\1\0\0\0\3\1\0\0\0\41\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\165\163\145\162\61\0\160\64\44\44\167\60\162\66\0" + end_stream: true + } +} +actions { + on_write { + data: "\30\0\0\0\3\103\122\105\101\124\105\40\104\101\124\101\102\101\123\105\40\155\171\163\161\154\144\142" + } +} +actions { + on_write { + data: "\30\0\0\5\3\103\122\105\101\124\105\40\104\101\124\101\102\101\123\105\40\155\171\163\161\154\144\142" + } +} +actions { + on_write { + data: "\1\0\0\0\4" + } +} +actions { + on_write { + data: "\7\0\0\4\1\1\0\0\0\1\0" + } +} +actions { + on_write { + data: "\7\0\0\4\1\1\0\0\0\1\0" + } +} +actions { + on_write { + end_stream: true + } +} +actions { + on_write { + data: "3" + } +} \ No newline at end of file diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_auth b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_auth new file mode 100644 index 000000000000..b63f559beb0f --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_auth @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_write { + data: "\x0\x0\x0\x14\x0\x0\x0\x0\x0\x0\x0\xa\x0\x0\x0\x0\x0\x0\x0\xc8\x0\x0\x0\x0" + } +} +actions { + on_write { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_connect b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_connect new file mode 100644 index 000000000000..b63f559beb0f --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_connect @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_write { + data: "\x0\x0\x0\x14\x0\x0\x0\x0\x0\x0\x0\xa\x0\x0\x0\x0\x0\x0\x0\xc8\x0\x0\x0\x0" + } +} +actions { + on_write { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_ping b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_ping new file mode 100644 index 000000000000..b63f559beb0f --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_ping @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_write { + data: "\x0\x0\x0\x14\x0\x0\x0\x0\x0\x0\x0\xa\x0\x0\x0\x0\x0\x0\x0\xc8\x0\x0\x0\x0" + } +} +actions { + on_write { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_control b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_control new file mode 100644 index 000000000000..9dd0c0046d3d --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_control @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_write { + data: "\x0\x0\x0\x10\xff\xff\xff\xf8\x0\x0\x0\x0\x0\x0\x7\xd0\x0\x0\x0\x0" + } +} +actions { + on_write { + } +} + diff --git a/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_event b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_event new file mode 100644 index 000000000000..9ff600715bba --- /dev/null +++ b/test/extensions/filters/network/common/fuzz/network_writefilter_corpus/zookeeper_proxy_watch_event @@ -0,0 +1,17 @@ +config { + name: "envoy.filters.network.zookeeper_proxy" + typed_config { + type_url: "type.googleapis.com/envoy.extensions.filters.network.zookeeper_proxy.v3.ZooKeeperProxy" + value: "\xa\xb\x74\x65\x73\x74\x5f\x70\x72\x65\x66\x69\x78" + } +} +actions { + on_write { + data: "\x0\x0\x0\x20\xff\xff\xff\xff\x0\x0\x0\x0\x0\x0\x3\xe8\x0\x0\x0\x0\x0\x0\x0\x1\x0\x0\x0\x0\x0\x0\x0\x4\x2f\x66\x6f\x6f" + } +} +actions { + on_write { + } +} + From a2cb83c6fea1bded33b6f9e56efdfa7384c25f0a Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 14 Aug 2020 11:16:11 -0700 Subject: [PATCH 66/67] test: fix http_timeout_integration_test flake (#12654) Fixes https://github.com/envoyproxy/envoy/issues/12653 Signed-off-by: Matt Klein --- test/integration/http_timeout_integration_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/http_timeout_integration_test.cc b/test/integration/http_timeout_integration_test.cc index 182c273918e1..77ea916b467a 100644 --- a/test/integration/http_timeout_integration_test.cc +++ b/test/integration/http_timeout_integration_test.cc @@ -282,7 +282,7 @@ TEST_P(HttpTimeoutIntegrationTest, PerTryTimeoutWithoutGlobalTimeout) { {"x-forwarded-for", "10.0.0.1"}, {"x-envoy-retry-on", "5xx"}, {"x-envoy-upstream-rq-timeout-ms", "0"}, - {"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}); + {"x-envoy-upstream-rq-per-try-timeout-ms", "50"}}); auto response = std::move(encoder_decoder.second); request_encoder_ = &encoder_decoder.first; @@ -294,11 +294,11 @@ TEST_P(HttpTimeoutIntegrationTest, PerTryTimeoutWithoutGlobalTimeout) { ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); // Trigger per try timeout (but not global timeout) and wait for reset. - timeSystem().advanceTimeWait(std::chrono::milliseconds(5)); + timeSystem().advanceTimeWait(std::chrono::milliseconds(50)); ASSERT_TRUE(upstream_request_->waitForReset()); // Wait for a second request to be sent upstream. Max retry backoff is 25ms so advance time that - // much. + // much. This is always less than the next request's per try timeout. timeSystem().advanceTimeWait(std::chrono::milliseconds(25)); ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); ASSERT_TRUE(upstream_request_->waitForHeadersComplete()); From b7138814dde530b8c2957e806ea40879a4fdce32 Mon Sep 17 00:00:00 2001 From: Jinhui Song Date: Fri, 14 Aug 2020 15:28:52 -0500 Subject: [PATCH 67/67] logger: support log control in admin interface and command line option for Fancy Logger (#12369) Add log control (list and modify log level) in admin interface for Fancy Logger, a new fine-grained logger for Envoy, and provide command line option --enable-fine-grain-logging for developers. Additional Description: A doc of overview is provided here: source/docs/fancy_logger.md. Risk Level: Medium Testing: Unit tests. Docs Changes: Added a new option --enable-fine-grain-logging and doc it. Release Notes: Added to current.rst. Signed-off-by: Jinhui Song --- api/envoy/admin/v3/server_info.proto | 5 +- api/envoy/admin/v4alpha/server_info.proto | 5 +- docs/root/operations/admin.rst | 4 +- docs/root/operations/cli.rst | 7 ++ docs/root/version_history/current.rst | 1 + .../envoy/admin/v3/server_info.proto | 5 +- .../envoy/admin/v4alpha/server_info.proto | 5 +- include/envoy/server/options.h | 5 ++ source/common/common/BUILD | 27 ++----- source/common/common/fancy_logger.cc | 23 +++++- source/common/common/fancy_logger.h | 14 +++- source/common/common/logger.cc | 48 ++++++++++-- source/common/common/logger.h | 73 ++++++++++++++----- source/docs/fancy_logger.md | 61 ++++++++++++++++ source/exe/main_common.cc | 3 +- source/server/admin/logs_handler.cc | 65 +++++++++++------ source/server/options_impl.cc | 5 ++ source/server/options_impl.h | 5 ++ test/common/common/BUILD | 19 +++-- test/common/common/log_macros_test.cc | 61 ++++++++++++---- test/mocks/server/options.h | 1 + test/server/admin/BUILD | 1 + test/server/admin/logs_handler_test.cc | 22 ++++++ test/server/options_impl_test.cc | 5 +- test/test_runner.cc | 3 +- 25 files changed, 378 insertions(+), 95 deletions(-) create mode 100644 source/docs/fancy_logger.md diff --git a/api/envoy/admin/v3/server_info.proto b/api/envoy/admin/v3/server_info.proto index 7f8ea45650d4..b91303f3d8fe 100644 --- a/api/envoy/admin/v3/server_info.proto +++ b/api/envoy/admin/v3/server_info.proto @@ -54,7 +54,7 @@ message ServerInfo { CommandLineOptions command_line_options = 6; } -// [#next-free-field: 34] +// [#next-free-field: 35] message CommandLineOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.admin.v2alpha.CommandLineOptions"; @@ -176,4 +176,7 @@ message CommandLineOptions { // See :option:`--bootstrap-version` for details. uint32 bootstrap_version = 29; + + // See :option:`--enable-fine-grain-logging` for details. + bool enable_fine_grain_logging = 34; } diff --git a/api/envoy/admin/v4alpha/server_info.proto b/api/envoy/admin/v4alpha/server_info.proto index e3e40ac2eabc..3f3570af0111 100644 --- a/api/envoy/admin/v4alpha/server_info.proto +++ b/api/envoy/admin/v4alpha/server_info.proto @@ -54,7 +54,7 @@ message ServerInfo { CommandLineOptions command_line_options = 6; } -// [#next-free-field: 34] +// [#next-free-field: 35] message CommandLineOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.admin.v3.CommandLineOptions"; @@ -175,4 +175,7 @@ message CommandLineOptions { // See :option:`--bootstrap-version` for details. uint32 bootstrap_version = 29; + + // See :option:`--enable-fine-grain-logging` for details. + bool enable_fine_grain_logging = 34; } diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index b2ad2e7c6391..d649197cc910 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -237,7 +237,9 @@ modify different aspects of the server: .. note:: - Generally only used during development. + Generally only used during development. With `--enable-fine-grain-logging` being set, the logger is represented + by the path of the file it belongs to (to be specific, the path determined by `__FILE__`), so the logger list + will show a list of file paths, and the specific path should be used as to change the log level. .. http:get:: /memory diff --git a/docs/root/operations/cli.rst b/docs/root/operations/cli.rst index 5cbd5b911051..243b8525580a 100644 --- a/docs/root/operations/cli.rst +++ b/docs/root/operations/cli.rst @@ -184,6 +184,13 @@ following are the command line options that Envoy supports. The :ref:`hot restart wrapper ` sets the *RESTART_EPOCH* environment variable which should be passed to this option in most cases. +.. option:: --enable-fine-grain-logging + *(optional)* Enables fine-grain logger with file level log control and runtime update at administration + interface. If enabled, main log macros including `ENVOY_LOG`, `ENVOY_CONN_LOG`, `ENVOY_STREAM_LOG` and + `ENVOY_FLUSH_LOG` will use a per-file logger, and the usage doesn't need `Envoy::Logger::Loggable` any + more. The administration interface usage is similar. Please see `Administration interface + `_ for more detail. + .. option:: --hot-restart-version *(optional)* Outputs an opaque hot restart compatibility version for the binary. This can be diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index 326d4d586f03..f1d42573eabf 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -21,6 +21,7 @@ Minor Behavior Changes * http: fixed the 100-continue response path to properly handle upstream failure by sending 5xx responses. This behavior can be temporarily reverted by setting `envoy.reloadable_features.allow_500_after_100` to false. * http: the per-stream FilterState maintained by the HTTP connection manager will now provide read/write access to the downstream connection FilterState. As such, code that relies on interacting with this might see a change in behavior. +* logging: add fine-grain logging for file level log control with logger management at administration interface. It can be enabled by option `--enable-fine-grain-logging`. * logging: change default log format to `"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"` and default value of :option:`--log-format-prefix-with-location` to `0`. * logging: nghttp2 log messages no longer appear at trace level unless `ENVOY_NGHTTP2_TRACE` is set in the environment. diff --git a/generated_api_shadow/envoy/admin/v3/server_info.proto b/generated_api_shadow/envoy/admin/v3/server_info.proto index 480ce862c6a5..320bc022a5d6 100644 --- a/generated_api_shadow/envoy/admin/v3/server_info.proto +++ b/generated_api_shadow/envoy/admin/v3/server_info.proto @@ -54,7 +54,7 @@ message ServerInfo { CommandLineOptions command_line_options = 6; } -// [#next-free-field: 34] +// [#next-free-field: 35] message CommandLineOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.admin.v2alpha.CommandLineOptions"; @@ -175,6 +175,9 @@ message CommandLineOptions { // See :option:`--bootstrap-version` for details. uint32 bootstrap_version = 29; + // See :option:`--enable-fine-grain-logging` for details. + bool enable_fine_grain_logging = 34; + uint64 hidden_envoy_deprecated_max_stats = 20 [deprecated = true, (envoy.annotations.disallowed_by_default) = true]; diff --git a/generated_api_shadow/envoy/admin/v4alpha/server_info.proto b/generated_api_shadow/envoy/admin/v4alpha/server_info.proto index e3e40ac2eabc..3f3570af0111 100644 --- a/generated_api_shadow/envoy/admin/v4alpha/server_info.proto +++ b/generated_api_shadow/envoy/admin/v4alpha/server_info.proto @@ -54,7 +54,7 @@ message ServerInfo { CommandLineOptions command_line_options = 6; } -// [#next-free-field: 34] +// [#next-free-field: 35] message CommandLineOptions { option (udpa.annotations.versioning).previous_message_type = "envoy.admin.v3.CommandLineOptions"; @@ -175,4 +175,7 @@ message CommandLineOptions { // See :option:`--bootstrap-version` for details. uint32 bootstrap_version = 29; + + // See :option:`--enable-fine-grain-logging` for details. + bool enable_fine_grain_logging = 34; } diff --git a/include/envoy/server/options.h b/include/envoy/server/options.h index 98ea52e2ce6c..546e0f9d8e80 100644 --- a/include/envoy/server/options.h +++ b/include/envoy/server/options.h @@ -178,6 +178,11 @@ class Options { */ virtual bool logFormatEscaped() const PURE; + /** + * @return const bool logger mode: whether to use Fancy Logger. + */ + virtual bool enableFineGrainLogging() const PURE; + /** * @return const std::string& the log file path. */ diff --git a/source/common/common/BUILD b/source/common/common/BUILD index b1c7c2d13240..8c645fd32a91 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -137,30 +137,19 @@ envoy_cc_library( # Contains minimal code for logging to stderr. envoy_cc_library( name = "minimal_logger_lib", - srcs = ["logger.cc"], - hdrs = ["logger.h"], - external_deps = ["abseil_synchronization"], - deps = [ - ":base_logger_lib", - ":lock_guard_lib", - ":macros", - ":non_copyable", - ] + select({ - "//bazel:android_logger": ["logger_impl_lib_android"], - "//conditions:default": ["logger_impl_lib_standard"], - }), -) - -envoy_cc_library( - name = "fancy_logger_lib", - srcs = ["fancy_logger.cc"], - hdrs = ["fancy_logger.h"], + srcs = [ + "fancy_logger.cc", + "logger.cc", + ], + hdrs = [ + "fancy_logger.h", + "logger.h", + ], external_deps = ["abseil_synchronization"], deps = [ ":base_logger_lib", ":lock_guard_lib", ":macros", - ":minimal_logger_lib", ":non_copyable", ] + select({ "//bazel:android_logger": ["logger_impl_lib_android"], diff --git a/source/common/common/fancy_logger.cc b/source/common/common/fancy_logger.cc index ef90afeefb98..a6c1ff215de0 100644 --- a/source/common/common/fancy_logger.cc +++ b/source/common/common/fancy_logger.cc @@ -26,7 +26,11 @@ class FancyBasicLockable : public Thread::BasicLockable { SpdLoggerSharedPtr FancyContext::getFancyLogEntry(std::string key) ABSL_LOCKS_EXCLUDED(fancy_log_lock_) { absl::ReaderMutexLock l(&fancy_log_lock_); - return fancy_log_map_->find(key)->second; + auto it = fancy_log_map_->find(key); + if (it != fancy_log_map_->end()) { + return it->second; + } + return nullptr; } void FancyContext::initFancyLogger(std::string key, std::atomic& logger) @@ -69,6 +73,23 @@ void FancyContext::setDefaultFancyLevelFormat(spdlog::level::level_enum level, s } } +std::string FancyContext::listFancyLoggers() ABSL_LOCKS_EXCLUDED(fancy_log_lock_) { + std::string info = ""; + absl::ReaderMutexLock l(&fancy_log_lock_); + for (const auto& it : *fancy_log_map_) { + info += fmt::format(" {}: {}\n", it.first, static_cast(it.second->level())); + } + return info; +} + +void FancyContext::setAllFancyLoggers(spdlog::level::level_enum level) + ABSL_LOCKS_EXCLUDED(fancy_log_lock_) { + absl::ReaderMutexLock l(&fancy_log_lock_); + for (const auto& it : *fancy_log_map_) { + it.second->set_level(level); + } +} + void FancyContext::initSink() { spdlog::sink_ptr sink = Logger::Registry::getSink(); Logger::DelegatingLogSinkSharedPtr sp = std::static_pointer_cast(sink); diff --git a/source/common/common/fancy_logger.h b/source/common/common/fancy_logger.h index dee92922d029..24cb55442164 100644 --- a/source/common/common/fancy_logger.h +++ b/source/common/common/fancy_logger.h @@ -38,11 +38,23 @@ class FancyContext { ABSL_LOCKS_EXCLUDED(fancy_log_lock_); /** - * Sets the default logger level and format when updating context. + * Sets the default logger level and format when updating context. It should only be used in + * Context, otherwise the fancy_default_level will possibly be inconsistent with the actual + * logger level. */ void setDefaultFancyLevelFormat(spdlog::level::level_enum level, std::string format) ABSL_LOCKS_EXCLUDED(fancy_log_lock_); + /** + * Lists keys and levels of all loggers in a string for admin page usage. + */ + std::string listFancyLoggers() ABSL_LOCKS_EXCLUDED(fancy_log_lock_); + + /** + * Sets the levels of all loggers. + */ + void setAllFancyLoggers(spdlog::level::level_enum level) ABSL_LOCKS_EXCLUDED(fancy_log_lock_); + private: /** * Initializes sink for the initialization of loggers, needed only in benchmark test. diff --git a/source/common/common/logger.cc b/source/common/common/logger.cc index 530484876202..e0f79783f88f 100644 --- a/source/common/common/logger.cc +++ b/source/common/common/logger.cc @@ -10,6 +10,7 @@ #include "absl/strings/ascii.h" #include "absl/strings/escaping.h" +#include "absl/strings/str_replace.h" #include "absl/strings/strip.h" #include "spdlog/spdlog.h" @@ -110,9 +111,9 @@ DelegatingLogSinkSharedPtr DelegatingLogSink::init() { static Context* current_context = nullptr; Context::Context(spdlog::level::level_enum log_level, const std::string& log_format, - Thread::BasicLockable& lock, bool should_escape) + Thread::BasicLockable& lock, bool should_escape, bool enable_fine_grain_logging) : log_level_(log_level), log_format_(log_format), lock_(lock), should_escape_(should_escape), - save_context_(current_context) { + enable_fine_grain_logging_(enable_fine_grain_logging), save_context_(current_context) { current_context = this; activate(); } @@ -126,21 +127,54 @@ Context::~Context() { } } -void Context::activate(LoggerMode mode) { +void Context::activate() { Registry::getSink()->setLock(lock_); Registry::getSink()->setShouldEscape(should_escape_); Registry::setLogLevel(log_level_); Registry::setLogFormat(log_format_); - if (mode == LoggerMode::Fancy) { - fancy_default_level_ = log_level_; - fancy_log_format_ = log_format_; + // sets level and format for Fancy Logger + fancy_default_level_ = log_level_; + fancy_log_format_ = log_format_; + if (enable_fine_grain_logging_) { + // loggers with default level before are set to log_level_ as new default + getFancyContext().setDefaultFancyLevelFormat(log_level_, log_format_); + if (log_format_ == Logger::Logger::DEFAULT_LOG_FORMAT) { + fancy_log_format_ = absl::StrReplaceAll(log_format_, {{"[%n]", ""}}); + } + } +} + +bool Context::useFancyLogger() { + if (current_context) { + return current_context->enable_fine_grain_logging_; + } + return false; +} + +void Context::enableFancyLogger() { + current_context->enable_fine_grain_logging_ = true; + if (current_context) { + getFancyContext().setDefaultFancyLevelFormat(current_context->log_level_, + current_context->log_format_); + current_context->fancy_default_level_ = current_context->log_level_; + current_context->fancy_log_format_ = current_context->log_format_; + if (current_context->log_format_ == Logger::Logger::DEFAULT_LOG_FORMAT) { + current_context->fancy_log_format_ = + absl::StrReplaceAll(current_context->log_format_, {{"[%n]", ""}}); + } + } +} + +void Context::disableFancyLogger() { + if (current_context) { + current_context->enable_fine_grain_logging_ = false; } } std::string Context::getFancyLogFormat() { if (!current_context) { // Context is not instantiated in benchmark test - return "[%Y-%m-%d %T.%e][%t][%l][%n] %v"; + return "[%Y-%m-%d %T.%e][%t][%l] %v"; } return current_context->fancy_log_format_; } diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 5c2ed08f497c..28827d44c71a 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -8,6 +8,7 @@ #include "envoy/thread/thread.h" #include "common/common/base_logger.h" +#include "common/common/fancy_logger.h" #include "common/common/fmt.h" #include "common/common/logger_impl.h" #include "common/common/macros.h" @@ -225,23 +226,36 @@ enum class LoggerMode { Envoy, Fancy }; * context is restored. When all contexts are destroyed, the lock is cleared, * and logging will remain unlocked, the same state it is in prior to * instantiating a Context. + * + * Settings for Fancy Logger, a file level logger without explicit implementation of + * Envoy::Logger:Loggable, are integrated here, as they should be updated when + * context switch occurs. */ class Context { public: Context(spdlog::level::level_enum log_level, const std::string& log_format, - Thread::BasicLockable& lock, bool should_escape); + Thread::BasicLockable& lock, bool should_escape, bool enable_fine_grain_logging = false); ~Context(); + /** + * Same as before, with boolean returned to use in log macros. + */ + static bool useFancyLogger(); + + static void enableFancyLogger(); + static void disableFancyLogger(); + static std::string getFancyLogFormat(); static spdlog::level::level_enum getFancyDefaultLevel(); private: - void activate(LoggerMode mode = LoggerMode::Envoy); + void activate(); const spdlog::level::level_enum log_level_; const std::string log_format_; Thread::BasicLockable& lock_; bool should_escape_; + bool enable_fine_grain_logging_; Context* const save_context_; std::string fancy_log_format_ = "[%Y-%m-%d %T.%e][%t][%l][%n] %v"; @@ -351,16 +365,6 @@ template class Loggable { */ #define ENVOY_LOGGER() __log_do_not_use_read_comment() -/** - * Convenience macro to flush logger. - */ -#define ENVOY_FLUSH_LOG() ENVOY_LOGGER().flush() - -/** - * Convenience macro to log to the class' logger. - */ -#define ENVOY_LOG(LEVEL, ...) ENVOY_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, ##__VA_ARGS__) - /** * Convenience macro to log to the misc logger, which allows for logging without of direct access to * a logger. @@ -374,9 +378,6 @@ template class Loggable { #define ENVOY_CONN_LOG_TO_LOGGER(LOGGER, LEVEL, FORMAT, CONNECTION, ...) \ ENVOY_LOG_TO_LOGGER(LOGGER, LEVEL, "[C{}] " FORMAT, (CONNECTION).id(), ##__VA_ARGS__) -#define ENVOY_CONN_LOG(LEVEL, FORMAT, CONNECTION, ...) \ - ENVOY_CONN_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, FORMAT, CONNECTION, ##__VA_ARGS__) - /** * Convenience macros for logging with a stream ID and a connection ID. */ @@ -385,9 +386,45 @@ template class Loggable { (STREAM).connection() ? (STREAM).connection()->id() : 0, \ (STREAM).streamId(), ##__VA_ARGS__) -#define ENVOY_STREAM_LOG(LEVEL, FORMAT, STREAM, ...) \ - ENVOY_STREAM_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, FORMAT, STREAM, ##__VA_ARGS__) - // TODO(danielhochman): macros(s)/function(s) for logging structures that support iteration. +/** + * Command line options for log macros: use Fancy Logger or not. + */ +#define ENVOY_LOG(LEVEL, ...) \ + do { \ + if (Envoy::Logger::Context::useFancyLogger()) { \ + FANCY_LOG(LEVEL, ##__VA_ARGS__); \ + } else { \ + ENVOY_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, ##__VA_ARGS__); \ + } \ + } while (0) + +#define ENVOY_FLUSH_LOG() \ + do { \ + if (Envoy::Logger::Context::useFancyLogger()) { \ + FANCY_FLUSH_LOG(); \ + } else { \ + ENVOY_LOGGER().flush(); \ + } \ + } while (0) + +#define ENVOY_CONN_LOG(LEVEL, FORMAT, CONNECTION, ...) \ + do { \ + if (Envoy::Logger::Context::useFancyLogger()) { \ + FANCY_CONN_LOG(LEVEL, FORMAT, CONNECTION, ##__VA_ARGS__); \ + } else { \ + ENVOY_CONN_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, FORMAT, CONNECTION, ##__VA_ARGS__); \ + } \ + } while (0) + +#define ENVOY_STREAM_LOG(LEVEL, FORMAT, STREAM, ...) \ + do { \ + if (Envoy::Logger::Context::useFancyLogger()) { \ + FANCY_STREAM_LOG(LEVEL, FORMAT, STREAM, ##__VA_ARGS__); \ + } else { \ + ENVOY_STREAM_LOG_TO_LOGGER(ENVOY_LOGGER(), LEVEL, FORMAT, STREAM, ##__VA_ARGS__); \ + } \ + } while (0) + } // namespace Envoy diff --git a/source/docs/fancy_logger.md b/source/docs/fancy_logger.md new file mode 100644 index 000000000000..2a8c2e665f51 --- /dev/null +++ b/source/docs/fancy_logger.md @@ -0,0 +1,61 @@ +## Fancy Logger: Flexible Granularity Log Control in Envoy + +### Overview +Fancy Logger is a logger with finer grained log level control and runtime logger update using administration interface. Compared to the existing logger in Envoy, Fancy Logger provides file level control which can be easily extended to finer grained logger with function or line level control, and it is completely automatic and never requires developers to explicitly specify the logging component. Besides, it has a comparable speed as Envoy's logger. + +### Basic Usage +The basic usage of Fancy Logger is to explicitly call its macros: +``` + FANCY_LOG(info, "Hello world! Here's a line of fancy log!"); + FANCY_LOG(error, "Fancy Error! Here's the second message!"); +``` +If the level of log message is higher than that of the file, macros above will print messages with the file name like this: +``` +[2020-07-29 22:27:02.594][15][error][test/common/common/log_macros_test.cc:149] Fancy Error! Here\'s the second message! +``` +More macros with connection and stream information: +``` + NiceMock connection_; + NiceMock stream_; + FANCY_CONN_LOG(warn, "Fake info {} of connection", connection_, 1); + FANCY_STREAM_LOG(warn, "Fake warning {} of stream", stream_, 1); +``` +To flush a logger, `FANCY_FLUSH_LOG()` can be used. + +### Enable Fancy Logger using Command Line Option +A command line option is provided to enable Fancy Logger: `--enable-fine-grain-logging`. It enables Fancy Logger for Envoy, i.e. replaces most Envoy's log macros (`ENVOY_LOG, ENVOY_FLUSH_LOG, ENVOY_CONN_LOG, ENVOY_STREAM_LOG`) with corresponding Fancy Logger's macros. + +If Fancy Logger is enabled, the default log format is `"[%Y-%m-%d %T.%e][%t][%l] [%g:%#] %v"`, where the logger name is omitted compared to Envoy's default as it's the same as file name. The default log level is info, if not specidfied by user of any logging context. + +Note that Envoy's logger can still be used in Fancy mode. These macros are not replaced: `GET_MISC_LOGGER, ENVOY_LOG_MISC, ENVOY_LOGGER, ENVOY_LOG_TO_LOGGER, ENVOY_CONN_LOG_TO_LOGGER, ENVOY_STREAM_LOG_TO_LOGGER`. For example, `ENVOY_LOG_LOGGER(ENVOY_LOGGER(), LEVEL, ...)` is equivalent to `ENVOY_LOG` in Envoy mode. + +If Fancy Logger is not enabled, existing Envoy's logger is used. In this mode, basic macros like `FANCY_LOG` can be used but the main part of `ENVOY_LOG` will keep the same. One limitation is that logger update in admin page is not supported by default as it detects Envoy mode. The reason is: Envoy mode is designed only to be back compatible. To address it, developers can use `Logger::Context::enableFancyLogger()` to manually enable Fancy Logger. + +### Runtime Update +Runtime update of Fancy Logger is supported with administration interface, i.e. admin page, and Fancy mode needs to be enabled to use it. Same as Envoy's logger, the following functionalities are provided: + +1. `POST /logging`: List all names (i.e. file paths) of all active loggers and their levels; +2. `POST /logging?=`: Given the current file path, change the log level of the file; +3. `POST /logging?level=`: Change levels of all loggers. + +Users can view and change the log level in a granularity of file in runtime through admin page. Note that `file_path` is determined by `__FILE__` macro, which is the path seen by preprocessor. + +### Implementation Details +Fancy Logger can be divided into two parts: +1. Core part: file level logger without explicit inheriting `Envoy::Logger::Loggable`; +2. Hook part: control interfaces, such as command line and admin page. + +#### Core Part +The core of Fancy Logger is implemented in `class FancyContext`, which is a singleton class. Filenames (i.e. keys) and logger pointers are stored in `FancyContext::fancy_log_map_`. There are several code paths when `FANCY_LOG` is called. + +1. Slow path: if `FANCY_LOG` is first called in a file, `FancyContext::initFancyLogger(key, local_logger_ptr)` is called to create a new logger globally and store the `` pair in `fancy_log_map_`. Then local logger pointer is updated and will never be changed. +2. Medium path: if `FANCY_LOG` is first called at the call site but not in the file, `FancyContext::initFancyLogger(key, local_logger_ptr)` is still called but local logger pointer is quickly set to the global logger, as there is already global record of the given filename. +3. Fast path: if `FANCY_LOG` is called after first call at the site, the log is directly printed using local logger pointer. + +#### Hook Part +Fancy Logger provides control interfaces through command line and admin page. + +To pass the arguments such as log format and default log level to Fancy Logger, `fancy_log_format` and `fancy_default_level` are added in `class Context` and they are updated when a new logging context is activated. `getFancyContext().setDefaultFancyLevelFormat(level, format)` is called in `Context::activate()` to set log format and update loggers' previous default level to the new level. + +To support the runtime update in admin page, log handler in admin page uses `getFancyContext().listFancyLoggers()` to show all Fancy Loggers, `getFancyContext().setFancyLogger(name, level)` to set a specific logger and `getFancyContext().setAllFancyLoggers(level)` to set all loggers. + diff --git a/source/exe/main_common.cc b/source/exe/main_common.cc index ace645ebb1aa..32b13d695306 100644 --- a/source/exe/main_common.cc +++ b/source/exe/main_common.cc @@ -68,7 +68,8 @@ MainCommonBase::MainCommonBase(const OptionsImpl& options, Event::TimeSystem& ti Thread::BasicLockable& access_log_lock = restarter_->accessLogLock(); auto local_address = Network::Utility::getLocalAddress(options_.localAddressIpVersion()); logging_context_ = std::make_unique(options_.logLevel(), options_.logFormat(), - log_lock, options_.logFormatEscaped()); + log_lock, options_.logFormatEscaped(), + options_.enableFineGrainLogging()); configureComponentLogLevels(); diff --git a/source/server/admin/logs_handler.cc b/source/server/admin/logs_handler.cc index 57b0fbdfca2f..353a51794745 100644 --- a/source/server/admin/logs_handler.cc +++ b/source/server/admin/logs_handler.cc @@ -2,6 +2,7 @@ #include +#include "common/common/fancy_logger.h" #include "common/common/logger.h" #include "server/admin/utils.h" @@ -28,12 +29,19 @@ Http::Code LogsHandler::handlerLogging(absl::string_view url, Http::ResponseHead rc = Http::Code::NotFound; } - response.add("active loggers:\n"); - for (const Logger::Logger& logger : Logger::Registry::loggers()) { - response.add(fmt::format(" {}: {}\n", logger.name(), logger.levelString())); + if (!Logger::Context::useFancyLogger()) { + response.add("active loggers:\n"); + for (const Logger::Logger& logger : Logger::Registry::loggers()) { + response.add(fmt::format(" {}: {}\n", logger.name(), logger.levelString())); + } + + response.add("\n"); + } else { + response.add("active loggers:\n"); + std::string logger_info = getFancyContext().listFancyLoggers(); + response.add(logger_info); } - response.add("\n"); return rc; } @@ -65,27 +73,40 @@ bool LogsHandler::changeLogLevel(const Http::Utility::QueryParams& params) { return false; } - // Now either change all levels or a single level. - if (name == "level") { - ENVOY_LOG(debug, "change all log levels: level='{}'", level); - for (Logger::Logger& logger : Logger::Registry::loggers()) { - logger.setLevel(static_cast(level_to_use)); - } - } else { - ENVOY_LOG(debug, "change log level: name='{}' level='{}'", name, level); - Logger::Logger* logger_to_change = nullptr; - for (Logger::Logger& logger : Logger::Registry::loggers()) { - if (logger.name() == name) { - logger_to_change = &logger; - break; + if (!Logger::Context::useFancyLogger()) { + // Now either change all levels or a single level. + if (name == "level") { + ENVOY_LOG(debug, "change all log levels: level='{}'", level); + for (Logger::Logger& logger : Logger::Registry::loggers()) { + logger.setLevel(static_cast(level_to_use)); + } + } else { + ENVOY_LOG(debug, "change log level: name='{}' level='{}'", name, level); + Logger::Logger* logger_to_change = nullptr; + for (Logger::Logger& logger : Logger::Registry::loggers()) { + if (logger.name() == name) { + logger_to_change = &logger; + break; + } } - } - if (!logger_to_change) { - return false; - } + if (!logger_to_change) { + return false; + } - logger_to_change->setLevel(static_cast(level_to_use)); + logger_to_change->setLevel(static_cast(level_to_use)); + } + } else { + // Level setting with Fancy Logger. + spdlog::level::level_enum lv = static_cast(level_to_use); + if (name == "level") { + FANCY_LOG(info, "change all log levels: level='{}'", level); + getFancyContext().setAllFancyLoggers(lv); + } else { + FANCY_LOG(info, "change log level: name='{}' level='{}'", name, level); + bool res = getFancyContext().setFancyLogger(name, lv); + return res; + } } return true; diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index fac2d8ae32c1..e287cc1e1a6c 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -108,6 +108,9 @@ OptionsImpl::OptionsImpl(std::vector args, TCLAP::SwitchArg log_format_escaped("", "log-format-escaped", "Escape c-style escape sequences in the application logs", cmd, false); + TCLAP::SwitchArg enable_fine_grain_logging( + "", "enable-fine-grain-logging", + "Logger mode: enable file level log control(Fancy Logger)or not", cmd, false); TCLAP::ValueArg log_format_prefix_with_location( "", "log-format-prefix-with-location", "Prefix all occurrences of '%v' in log format with with '[%g:%#] ' ('[path/to/file.cc:99] " @@ -191,6 +194,7 @@ OptionsImpl::OptionsImpl(std::vector args, log_format_ = absl::StrReplaceAll(log_format_, {{"%%", "%%"}, {"%v", "[%g:%#] %v"}}); } log_format_escaped_ = log_format_escaped.getValue(); + enable_fine_grain_logging_ = enable_fine_grain_logging.getValue(); parseComponentLogLevels(component_log_level.getValue()); @@ -354,6 +358,7 @@ Server::CommandLineOptionsPtr OptionsImpl::toCommandLineOptions() const { spdlog::level::to_string_view(logLevel()).size()); command_line_options->set_log_format(logFormat()); command_line_options->set_log_format_escaped(logFormatEscaped()); + command_line_options->set_enable_fine_grain_logging(enableFineGrainLogging()); command_line_options->set_log_path(logPath()); command_line_options->set_service_cluster(serviceClusterName()); command_line_options->set_service_node(serviceNodeName()); diff --git a/source/server/options_impl.h b/source/server/options_impl.h index bb8fd78eaadd..ad88e71f6c10 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -133,6 +133,7 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable disabled_extensions_; uint32_t count_; + + // Initialization added here to avoid integration_admin_test failure caused by uninitialized + // enable_fine_grain_logging_. + bool enable_fine_grain_logging_ = false; }; /** diff --git a/test/common/common/BUILD b/test/common/common/BUILD index 21ea6fe21df9..20392d66be19 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -114,7 +114,19 @@ envoy_cc_test( name = "log_macros_test", srcs = ["log_macros_test.cc"], deps = [ - "//source/common/common:fancy_logger_lib", + "//source/common/common:minimal_logger_lib", + "//test/mocks/http:http_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/upstream:upstream_mocks", + "//test/test_common:logging_lib", + ], +) + +envoy_cc_test( + name = "fancy_log_macros_test", + srcs = ["log_macros_test.cc"], + args = ["--enable-fine-grain-logging"], + deps = [ "//source/common/common:minimal_logger_lib", "//test/mocks/http:http_mocks", "//test/mocks/network:network_mocks", @@ -127,10 +139,7 @@ envoy_cc_benchmark_binary( name = "logger_speed_test", srcs = ["logger_speed_test.cc"], external_deps = ["benchmark"], - deps = [ - "//source/common/common:fancy_logger_lib", - "//source/common/common:minimal_logger_lib", - ], + deps = ["//source/common/common:minimal_logger_lib"], ) envoy_benchmark_test( diff --git a/test/common/common/log_macros_test.cc b/test/common/common/log_macros_test.cc index c19fbebfde01..01b08820be16 100644 --- a/test/common/common/log_macros_test.cc +++ b/test/common/common/log_macros_test.cc @@ -23,6 +23,8 @@ class TestFilterLog : public Logger::Loggable { ENVOY_LOG(critical, "fake message"); ENVOY_CONN_LOG(info, "fake message", connection_); ENVOY_STREAM_LOG(info, "fake message", stream_); + ENVOY_CONN_LOG(error, "fake error", connection_); + ENVOY_STREAM_LOG(error, "fake error", stream_); } void logMessageEscapeSequences() { ENVOY_LOG_MISC(info, "line 1 \n line 2 \t tab \\r test"); } @@ -155,30 +157,61 @@ TEST(Fancy, Global) { FANCY_FLUSH_LOG(); } +TEST(Fancy, FastPath) { + getFancyContext().setFancyLogger(__FILE__, spdlog::level::info); + for (int i = 0; i < 10; i++) { + FANCY_LOG(warn, "Fake warning No. {}", i); + } +} + TEST(Fancy, SetLevel) { const char* file = "P=NP_file"; - getFancyContext().setFancyLogger(file, spdlog::level::trace); + bool res = getFancyContext().setFancyLogger(file, spdlog::level::trace); + EXPECT_EQ(res, false); + SpdLoggerSharedPtr p = getFancyContext().getFancyLogEntry(file); + EXPECT_EQ(p, nullptr); - getFancyContext().setFancyLogger(__FILE__, spdlog::level::err); + res = getFancyContext().setFancyLogger(__FILE__, spdlog::level::err); + EXPECT_EQ(res, true); FANCY_LOG(error, "Fancy Error! Here's a test for level."); FANCY_LOG(warn, "Warning: you shouldn't see this message!"); + p = getFancyContext().getFancyLogEntry(__FILE__); + EXPECT_NE(p, nullptr); + EXPECT_EQ(p->level(), spdlog::level::err); + + getFancyContext().setAllFancyLoggers(spdlog::level::info); + FANCY_LOG(info, "Info: all loggers back to info."); + FANCY_LOG(debug, "Debug: you shouldn't see this message!"); + EXPECT_EQ(getFancyContext().getFancyLogEntry(__FILE__)->level(), spdlog::level::info); } -TEST(Fancy, Default) { - getFancyContext().setFancyLogger(__FILE__, spdlog::level::info); // revert to default - std::string fmt = "[%t][%l][%n] %v"; - getFancyContext().setDefaultFancyLevelFormat(spdlog::level::warn, fmt); - FANCY_LOG(info, "Info: you shouldn't see this message!"); - FANCY_LOG(warn, "Warning: warning at default log level!"); - EXPECT_EQ(Logger::Context::getFancyLogFormat(), "[%Y-%m-%d %T.%e][%t][%l][%n] %v"); - EXPECT_EQ(Logger::Context::getFancyDefaultLevel(), spdlog::level::info); +TEST(Fancy, Iteration) { + FANCY_LOG(info, "Info: iteration test begins."); + getFancyContext().setAllFancyLoggers(spdlog::level::info); + std::string output = getFancyContext().listFancyLoggers(); + EXPECT_EQ(output, " test/common/common/log_macros_test.cc: 2\n"); + std::string log_format = "[%T.%e][%t][%l][%n] %v"; + getFancyContext().setFancyLogger(__FILE__, spdlog::level::err); + // setDefaultFancyLevelFormat relies on previous default and might cause error online + // getFancyContext().setDefaultFancyLevelFormat(spdlog::level::warn, log_format); + FANCY_LOG(warn, "Warning: now level is warning, format changed (Date removed)."); + FANCY_LOG(warn, getFancyContext().listFancyLoggers()); + // EXPECT_EQ(getFancyContext().getFancyLogEntry(__FILE__)->level(), + // spdlog::level::warn); // note fancy_default_level isn't changed } -TEST(Fancy, FastPath) { - getFancyContext().setFancyLogger(__FILE__, spdlog::level::info); - for (int i = 0; i < 10; i++) { - FANCY_LOG(warn, "Fake warning No. {}", i); +TEST(Fancy, Context) { + FANCY_LOG(info, "Info: context API needs test."); + bool enable_fine_grain_logging = Logger::Context::useFancyLogger(); + printf(" --> If use fancy logger: %d\n", enable_fine_grain_logging); + if (enable_fine_grain_logging) { + FANCY_LOG(critical, "Cmd option set: all previous Envoy Log should be converted now!"); } + Logger::Context::enableFancyLogger(); + EXPECT_EQ(Logger::Context::useFancyLogger(), true); + EXPECT_EQ(Logger::Context::getFancyLogFormat(), "[%Y-%m-%d %T.%e][%t][%l] [%g:%#] %v"); + // EXPECT_EQ(Logger::Context::getFancyDefaultLevel(), + // spdlog::level::err); // default is error in test environment } } // namespace Envoy diff --git a/test/mocks/server/options.h b/test/mocks/server/options.h index 31a6112dca35..b4591ccbe828 100644 --- a/test/mocks/server/options.h +++ b/test/mocks/server/options.h @@ -36,6 +36,7 @@ class MockOptions : public Options { componentLogLevels, (), (const)); MOCK_METHOD(const std::string&, logFormat, (), (const)); MOCK_METHOD(bool, logFormatEscaped, (), (const)); + MOCK_METHOD(bool, enableFineGrainLogging, (), (const)); MOCK_METHOD(const std::string&, logPath, (), (const)); MOCK_METHOD(uint64_t, restartEpoch, (), (const)); MOCK_METHOD(std::chrono::milliseconds, fileFlushIntervalMsec, (), (const)); diff --git a/test/server/admin/BUILD b/test/server/admin/BUILD index b2af4c15f6f0..cd80379eec1e 100644 --- a/test/server/admin/BUILD +++ b/test/server/admin/BUILD @@ -91,6 +91,7 @@ envoy_cc_test( srcs = ["logs_handler_test.cc"], deps = [ ":admin_instance_lib", + "//source/common/common:minimal_logger_lib", ], ) diff --git a/test/server/admin/logs_handler_test.cc b/test/server/admin/logs_handler_test.cc index 9fc99c0c6225..3103164bb24e 100644 --- a/test/server/admin/logs_handler_test.cc +++ b/test/server/admin/logs_handler_test.cc @@ -1,3 +1,6 @@ +#include "common/common/fancy_logger.h" +#include "common/common/logger.h" + #include "test/server/admin/admin_instance.h" namespace Envoy { @@ -17,5 +20,24 @@ TEST_P(AdminInstanceTest, ReopenLogs) { EXPECT_EQ(Http::Code::OK, postCallback("/reopen_logs", header_map, response)); } +TEST_P(AdminInstanceTest, LogLevelSetting) { + Http::TestResponseHeaderMapImpl header_map; + Buffer::OwnedImpl response; + + // now for Envoy, w/o setting the mode + FANCY_LOG(info, "Build the logger for this file."); + Logger::Context::enableFancyLogger(); + postCallback("/logging", header_map, response); + FANCY_LOG(error, response.toString()); + + postCallback("/logging?level=warning", header_map, response); + FANCY_LOG(warn, "After post 1: all level is warning now!"); + EXPECT_EQ(getFancyContext().getFancyLogEntry(__FILE__)->level(), spdlog::level::warn); + std::string query = fmt::format("/logging?{}=info", __FILE__); + postCallback(query, header_map, response); + FANCY_LOG(info, "After post 2: level for this file is info now!"); + EXPECT_EQ(getFancyContext().getFancyLogEntry(__FILE__)->level(), spdlog::level::info); +} + } // namespace Server } // namespace Envoy diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 3898ffff14e3..04343efc658c 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -84,7 +84,9 @@ TEST_F(OptionsImplTest, All) { "--local-address-ip-version v6 -l info --component-log-level upstream:debug,connection:trace " "--service-cluster cluster --service-node node --service-zone zone " "--file-flush-interval-msec 9000 " - "--drain-time-s 60 --log-format [%v] --parent-shutdown-time-s 90 --log-path /foo/bar " + "--drain-time-s 60 --log-format [%v] --enable-fine-grain-logging --parent-shutdown-time-s 90 " + "--log-path " + "/foo/bar " "--disable-hot-restart --cpuset-threads --allow-unknown-static-fields " "--reject-unknown-dynamic-fields --use-fake-symbol-table 0 --base-id 5 " "--use-dynamic-base-id --base-id-path /foo/baz"); @@ -98,6 +100,7 @@ TEST_F(OptionsImplTest, All) { EXPECT_EQ(2, options->componentLogLevels().size()); EXPECT_EQ("[%v]", options->logFormat()); EXPECT_EQ("/foo/bar", options->logPath()); + EXPECT_EQ(true, options->enableFineGrainLogging()); EXPECT_EQ("cluster", options->serviceClusterName()); EXPECT_EQ("node", options->serviceNodeName()); EXPECT_EQ("zone", options->serviceZone()); diff --git a/test/test_runner.cc b/test/test_runner.cc index c90555899f1a..727268ffc4f1 100644 --- a/test/test_runner.cc +++ b/test/test_runner.cc @@ -140,7 +140,8 @@ int TestRunner::RunTests(int argc, char** argv) { Thread::MutexBasicLockable lock; Server::Options& options = TestEnvironment::getOptions(); - Logger::Context logging_state(options.logLevel(), options.logFormat(), lock, false); + Logger::Context logging_state(options.logLevel(), options.logFormat(), lock, false, + options.enableFineGrainLogging()); // Allocate fake log access manager. testing::NiceMock access_log_manager;