Merge pull request #5544

d046ca1 difficulty: fix check_hash on big endian (moneromooo-monero)
bdda084 epee: fix local/loopback checks on big endian (moneromooo-monero)
32c3834 storages: fix writing varints on big endian (moneromooo-monero)
516f7b9 storages: fix 'portable' storage on big endian (moneromooo-monero)
0e2fda5 unit_tests: fix levin unit test on big endian (moneromooo-monero)
4672b5c db_lmdb: print percentages as percentages, not ratios (moneromooo-monero)
54fd97a slow-hash: fix CNv2+ on big endian (moneromooo-monero)
c1fa4a7 boost: fix little/big endian compatibility (moneromooo-monero)
bc1144e Fix IP address serialization on big endian (moneromooo-monero)
This commit is contained in:
luigi1111 2019-09-08 19:38:58 -05:00
commit 48d8475b6e
No known key found for this signature in database
GPG Key ID: F4ACA0183641E010
15 changed files with 175 additions and 43 deletions

View File

@ -129,9 +129,12 @@ static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uin
return remainder;
}
#define IDENT16(x) ((uint16_t) (x))
#define IDENT32(x) ((uint32_t) (x))
#define IDENT64(x) ((uint64_t) (x))
#define SWAP16(x) ((((uint16_t) (x) & 0x00ff) << 8) | \
(((uint16_t) (x) & 0xff00) >> 8))
#define SWAP32(x) ((((uint32_t) (x) & 0x000000ff) << 24) | \
(((uint32_t) (x) & 0x0000ff00) << 8) | \
(((uint32_t) (x) & 0x00ff0000) >> 8) | \
@ -145,10 +148,18 @@ static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uin
(((uint64_t) (x) & 0x00ff000000000000) >> 40) | \
(((uint64_t) (x) & 0xff00000000000000) >> 56))
static inline uint16_t ident16(uint16_t x) { return x; }
static inline uint32_t ident32(uint32_t x) { return x; }
static inline uint64_t ident64(uint64_t x) { return x; }
#ifndef __OpenBSD__
# if defined(__ANDROID__) && defined(__swap16) && !defined(swap16)
# define swap16 __swap16
# elif !defined(swap16)
static inline uint16_t swap16(uint16_t x) {
return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8);
}
# endif
# if defined(__ANDROID__) && defined(__swap32) && !defined(swap32)
# define swap32 __swap32
# elif !defined(swap32)
@ -176,6 +187,12 @@ static inline uint64_t swap64(uint64_t x) {
static inline void mem_inplace_ident(void *mem UNUSED, size_t n UNUSED) { }
#undef UNUSED
static inline void mem_inplace_swap16(void *mem, size_t n) {
size_t i;
for (i = 0; i < n; i++) {
((uint16_t *) mem)[i] = swap16(((const uint16_t *) mem)[i]);
}
}
static inline void mem_inplace_swap32(void *mem, size_t n) {
size_t i;
for (i = 0; i < n; i++) {
@ -189,6 +206,9 @@ static inline void mem_inplace_swap64(void *mem, size_t n) {
}
}
static inline void memcpy_ident16(void *dst, const void *src, size_t n) {
memcpy(dst, src, 2 * n);
}
static inline void memcpy_ident32(void *dst, const void *src, size_t n) {
memcpy(dst, src, 4 * n);
}
@ -196,6 +216,12 @@ static inline void memcpy_ident64(void *dst, const void *src, size_t n) {
memcpy(dst, src, 8 * n);
}
static inline void memcpy_swap16(void *dst, const void *src, size_t n) {
size_t i;
for (i = 0; i < n; i++) {
((uint16_t *) dst)[i] = swap16(((const uint16_t *) src)[i]);
}
}
static inline void memcpy_swap32(void *dst, const void *src, size_t n) {
size_t i;
for (i = 0; i < n; i++) {
@ -220,6 +246,14 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e
#endif
#if BYTE_ORDER == LITTLE_ENDIAN
#define SWAP16LE IDENT16
#define SWAP16BE SWAP16
#define swap16le ident16
#define swap16be swap16
#define mem_inplace_swap16le mem_inplace_ident
#define mem_inplace_swap16be mem_inplace_swap16
#define memcpy_swap16le memcpy_ident16
#define memcpy_swap16be memcpy_swap16
#define SWAP32LE IDENT32
#define SWAP32BE SWAP32
#define swap32le ident32
@ -239,6 +273,14 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e
#endif
#if BYTE_ORDER == BIG_ENDIAN
#define SWAP16BE IDENT16
#define SWAP16LE SWAP16
#define swap16be ident16
#define swap16le swap16
#define mem_inplace_swap16be mem_inplace_ident
#define mem_inplace_swap16le mem_inplace_swap16
#define memcpy_swap16be memcpy_ident16
#define memcpy_swap16le memcpy_swap16
#define SWAP32BE IDENT32
#define SWAP32LE SWAP32
#define swap32be ident32

View File

@ -30,6 +30,11 @@
#include <string>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/asio/ip/address_v6.hpp>
#include "int-util.h"
// IP addresses are kept in network byte order
// Masks below are little endian
// -> convert from network byte order to host byte order before comparing
namespace epee
{
@ -62,6 +67,7 @@ namespace epee
inline
bool is_ip_local(uint32_t ip)
{
ip = SWAP32LE(ip);
/*
local ip area
10.0.0.0 10.255.255.255
@ -85,6 +91,7 @@ namespace epee
inline
bool is_ip_loopback(uint32_t ip)
{
ip = SWAP32LE(ip);
if( (ip | 0xffffff00) == 0xffffff7f)
return true;
//MAKE_IP

View File

@ -38,6 +38,7 @@
#include "enums.h"
#include "misc_log_ex.h"
#include "serialization/keyvalue_serialization.h"
#include "int-util.h"
#undef MONERO_DEFAULT_LOG_CATEGORY
#define MONERO_DEFAULT_LOG_CATEGORY "net"
@ -91,7 +92,20 @@ namespace net_utils
static constexpr bool is_blockable() noexcept { return true; }
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(m_ip)
if (is_store)
{
KV_SERIALIZE_VAL_POD_AS_BLOB_N(m_ip, "ip")
uint32_t ip = SWAP32LE(this_ref.m_ip);
epee::serialization::selector<is_store>::serialize(ip, stg, hparent_section, "m_ip");
}
else
{
if (!epee::serialization::selector<is_store>::serialize_t_val_as_blob(this_ref.m_ip, stg, hparent_section, "ip"))
{
KV_SERIALIZE(m_ip)
const_cast<ipv4_network_address&>(this_ref).m_ip = SWAP32LE(this_ref.m_ip);
}
}
KV_SERIALIZE(m_port)
END_KV_SERIALIZE_MAP()
};

View File

@ -0,0 +1,46 @@
// Copyright (c) 2019, The Monero Project
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include "int-util.h"
template<typename T> T convert_swapper(T t) { return t; }
template<> inline uint16_t convert_swapper(uint16_t t) { return SWAP16LE(t); }
template<> inline int16_t convert_swapper(int16_t t) { return SWAP16LE((uint16_t&)t); }
template<> inline uint32_t convert_swapper(uint32_t t) { return SWAP32LE(t); }
template<> inline int32_t convert_swapper(int32_t t) { return SWAP32LE((uint32_t&)t); }
template<> inline uint64_t convert_swapper(uint64_t t) { return SWAP64LE(t); }
template<> inline int64_t convert_swapper(int64_t t) { return SWAP64LE((uint64_t&)t); }
template<> inline double convert_swapper(double t) { union { uint64_t u; double d; } u; u.d = t; u.u = SWAP64LE(u.u); return u.d; }
#if BYTE_ORDER == BIG_ENDIAN
#define CONVERT_POD(x) convert_swapper(x)
#else
#define CONVERT_POD(x) (x)
#endif

View File

@ -30,6 +30,7 @@
#include "misc_language.h"
#include "portable_storage_base.h"
#include "portable_storage_bin_utils.h"
#ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
#define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
@ -117,6 +118,7 @@ namespace epee
RECURSION_LIMITATION();
static_assert(std::is_pod<t_pod_type>::value, "POD type expected");
read(&pod_val, sizeof(pod_val));
pod_val = CONVERT_POD(pod_val);
}
template<class t_type>

View File

@ -31,6 +31,7 @@
#include "pragma_comp_defs.h"
#include "misc_language.h"
#include "portable_storage_base.h"
#include "portable_storage_bin_utils.h"
namespace epee
{
@ -40,8 +41,9 @@ namespace epee
template<class pack_value, class t_stream>
size_t pack_varint_t(t_stream& strm, uint8_t type_or, size_t& pv)
{
pack_value v = (*((pack_value*)&pv)) << 2;
pack_value v = pv << 2;
v |= type_or;
v = CONVERT_POD(v);
strm.write((const char*)&v, sizeof(pack_value));
return sizeof(pack_value);
}
@ -93,8 +95,11 @@ namespace epee
uint8_t type = contained_type|SERIALIZE_FLAG_ARRAY;
m_strm.write((const char*)&type, 1);
pack_varint(m_strm, arr_pod.m_array.size());
for(const t_pod_type& x: arr_pod.m_array)
for(t_pod_type x: arr_pod.m_array)
{
x = CONVERT_POD(x);
m_strm.write((const char*)&x, sizeof(t_pod_type));
}
return true;
}
@ -147,7 +152,8 @@ namespace epee
bool pack_pod_type(uint8_t type, const pod_type& v)
{
m_strm.write((const char*)&type, 1);
m_strm.write((const char*)&v, sizeof(pod_type));
pod_type v0 = CONVERT_POD(v);
m_strm.write((const char*)&v0, sizeof(pod_type));
return true;
}
//section, array_entry

View File

@ -258,7 +258,7 @@ portable_binary_iarchive::load_impl(boost::intmax_t & l, char maxsize){
this->primitive_base_t::load_binary(cptr, size);
#if BOOST_ENDIAN_BIG_BYTE
if(m_flags & endian_little)
if((m_flags & endian_little) || (!(m_flags & endian_big)))
#else
if(m_flags & endian_big)
#endif
@ -343,6 +343,8 @@ portable_binary_iarchive::init(unsigned int flags){
);
#endif
}
if (!(m_flags & (endian_little | endian_big)))
m_flags |= endian_little;
unsigned char x;
load(x);
m_flags = x << CHAR_BIT;

View File

@ -171,7 +171,7 @@ protected:
void init(unsigned int flags);
public:
portable_binary_oarchive(std::ostream & os, unsigned flags = 0) :
portable_binary_oarchive(std::ostream & os, unsigned flags = endian_little) :
primitive_base_t(
* os.rdbuf(),
0 != (flags & boost::archive::no_codecvt)

View File

@ -595,7 +595,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
MDEBUG("Space remaining: " << mei.me_mapsize - size_used);
MDEBUG("Size threshold: " << threshold_size);
float resize_percent = RESIZE_PERCENT;
MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent);
MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % (100.*size_used/mei.me_mapsize) % (100.*resize_percent));
if (threshold_size > 0)
{

View File

@ -136,8 +136,8 @@ static inline int use_v4_jit(void)
{ \
U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \
U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \
division_result = state.hs.w[12]; \
sqrt_result = state.hs.w[13]; \
division_result = SWAP64LE(state.hs.w[12]); \
sqrt_result = SWAP64LE(state.hs.w[13]); \
} while (0)
#define VARIANT2_PORTABLE_INIT() \
@ -210,7 +210,7 @@ static inline int use_v4_jit(void)
uint64_t b0[2]; \
memcpy_swap64le(b0, b, 2); \
chunk2[0] = SWAP64LE(chunk1_old[0] + b0[0]); \
chunk2[1] = SWAP64LE(SWAP64LE(chunk1_old[1]) + b0[1]); \
chunk2[1] = SWAP64LE(chunk1_old[1] + b0[1]); \
if (variant >= 4) \
{ \
uint64_t out_copy[2]; \

View File

@ -34,7 +34,6 @@
#include <vector>
#include <string>
#include <boost/multiprecision/cpp_int.hpp>
#include "crypto/hash.h"
namespace cryptonote

View File

@ -95,7 +95,9 @@ namespace boost
{
uint32_t ip{na.ip()};
uint16_t port{na.port()};
ip = SWAP32LE(ip);
a & ip;
ip = SWAP32LE(ip);
a & port;
if (!typename Archive::is_saving())
na = epee::net_utils::ipv4_network_address{ip, port};

View File

@ -29,6 +29,7 @@
#pragma once
#include "string_tools.h"
#include "int-util.h"
#include "cryptonote_basic/difficulty.h"
template<uint64_t hash_target_high, uint64_t hash_target_low, uint64_t difficulty_high, uint64_t difficulty_low>
@ -44,13 +45,18 @@ public:
difficulty = difficulty_high;
difficulty = (difficulty << 64) | difficulty_low;
boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target;
((uint64_t*)&hash)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
uint64_t val;
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&hash)[0] = SWAP64LE(val);
hash_value >>= 64;
((uint64_t*)&hash)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&hash)[1] = SWAP64LE(val);
hash_value >>= 64;
((uint64_t*)&hash)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&hash)[2] = SWAP64LE(val);
hash_value >>= 64;
((uint64_t*)&hash)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&hash)[3] = SWAP64LE(val);
return true;
}

View File

@ -27,6 +27,7 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "gtest/gtest.h"
#include "int-util.h"
#include "cryptonote_basic/difficulty.h"
static cryptonote::difficulty_type MKDIFF(uint64_t high, uint64_t low)
@ -42,13 +43,18 @@ static crypto::hash MKHASH(uint64_t high, uint64_t low)
hash_target = (hash_target << 64) | low;
boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target;
crypto::hash h;
((uint64_t*)&h)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
uint64_t val;
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&h)[0] = SWAP64LE(val);
hash_value >>= 64;
((uint64_t*)&h)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&h)[1] = SWAP64LE(val);
hash_value >>= 64;
((uint64_t*)&h)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&h)[2] = SWAP64LE(val);
hash_value >>= 64;
((uint64_t*)&h)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>();
((uint64_t*)&h)[3] = SWAP64LE(val);
return h;
}

View File

@ -241,13 +241,13 @@ namespace
m_in_data.assign(256, 't');
m_req_head.m_signature = LEVIN_SIGNATURE;
m_req_head.m_cb = m_in_data.size();
m_req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
m_req_head.m_cb = SWAP64LE(m_in_data.size());
m_req_head.m_have_to_return_data = true;
m_req_head.m_command = expected_command;
m_req_head.m_return_code = LEVIN_OK;
m_req_head.m_flags = LEVIN_PACKET_REQUEST;
m_req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
m_req_head.m_command = SWAP32LE(expected_command);
m_req_head.m_return_code = SWAP32LE(LEVIN_OK);
m_req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
m_req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
m_commands_handler.return_code(expected_return_code);
m_commands_handler.invoke_out_buf(m_expected_invoke_out_buf);
@ -337,12 +337,12 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process
std::string in_data(256, 'q');
epee::levin::bucket_head2 req_head;
req_head.m_signature = LEVIN_SIGNATURE;
req_head.m_cb = in_data.size();
req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
req_head.m_cb = SWAP64LE(in_data.size());
req_head.m_have_to_return_data = true;
req_head.m_command = expected_command;
req_head.m_flags = LEVIN_PACKET_REQUEST;
req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
req_head.m_command = SWAP32LE(expected_command);
req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
std::string buf(reinterpret_cast<const char*>(&req_head), sizeof(req_head));
buf += in_data;
@ -373,13 +373,13 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process
// Check sent response
ASSERT_EQ(expected_out_data, out_data);
ASSERT_EQ(LEVIN_SIGNATURE, resp_head.m_signature);
ASSERT_EQ(expected_command, resp_head.m_command);
ASSERT_EQ(expected_return_code, resp_head.m_return_code);
ASSERT_EQ(expected_out_data.size(), resp_head.m_cb);
ASSERT_EQ(LEVIN_SIGNATURE, SWAP64LE(resp_head.m_signature));
ASSERT_EQ(expected_command, SWAP32LE(resp_head.m_command));
ASSERT_EQ(expected_return_code, SWAP32LE(resp_head.m_return_code));
ASSERT_EQ(expected_out_data.size(), SWAP64LE(resp_head.m_cb));
ASSERT_FALSE(resp_head.m_have_to_return_data);
ASSERT_EQ(LEVIN_PROTOCOL_VER_1, resp_head.m_protocol_version);
ASSERT_TRUE(0 != (resp_head.m_flags & LEVIN_PACKET_RESPONSE));
ASSERT_EQ(SWAP32LE(LEVIN_PROTOCOL_VER_1), resp_head.m_protocol_version);
ASSERT_TRUE(0 != (SWAP32LE(resp_head.m_flags) & LEVIN_PACKET_RESPONSE));
}
TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_processes_handle_read_as_notify)
@ -392,12 +392,12 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process
std::string in_data(256, 'e');
epee::levin::bucket_head2 req_head;
req_head.m_signature = LEVIN_SIGNATURE;
req_head.m_cb = in_data.size();
req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
req_head.m_cb = SWAP64LE(in_data.size());
req_head.m_have_to_return_data = false;
req_head.m_command = expected_command;
req_head.m_flags = LEVIN_PACKET_REQUEST;
req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
req_head.m_command = SWAP32LE(expected_command);
req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
std::string buf(reinterpret_cast<const char*>(&req_head), sizeof(req_head));
buf += in_data;
@ -618,7 +618,7 @@ TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_two_re
TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_unexpected_response)
{
m_req_head.m_flags = LEVIN_PACKET_RESPONSE;
m_req_head.m_flags = SWAP32LE(LEVIN_PACKET_RESPONSE);
prepare_buf();
ASSERT_FALSE(m_conn->m_protocol_handler.handle_recv(m_buf.data(), m_buf.size()));